test: deflake recent CI failures (#4139)

3 files changed, 42 insertions, 11 deletions

diff --git a/scripts/docs/preview.integration.test.ts b/scripts/docs/preview.integration.test.ts
index b74ac4ec..65c69f4d 100644
--- a/scripts/docs/preview.integration.test.ts
+++ b/scripts/docs/preview.integration.test.ts
@@ -14,8 +14,11 @@ const fixtureTimeoutMs = process.platform === 'win32' ? 120_000 : 30_000;
 
 afterEach(async () => {
   await Promise.all(
-    temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })),
+    temporaryDirectories.map((path) =>
+      rm(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }),
+    ),
   );
+  temporaryDirectories.length = 0;
 });
 
 describe('preview documentation snapshots', () => {
diff --git a/scripts/package_windows_conpty.py b/scripts/package_windows_conpty.py
index 2fc51a1e..3c63312e 100644
--- a/scripts/package_windows_conpty.py
+++ b/scripts/package_windows_conpty.py
@@ -7,6 +7,8 @@ import json
 import shutil
 import struct
 import tempfile
+import time
+import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
 import zipfile
@@ -74,10 +76,18 @@ def validate_nuspec(archive: zipfile.ZipFile, package: dict[str, Any]) -> None:
 def acquire_package(package: dict[str, Any], package_path: Path) -> None:
     package_path.parent.mkdir(parents=True, exist_ok=True)
     if not package_path.exists():
-        with urllib.request.urlopen(
-            package["url"], timeout=DOWNLOAD_TIMEOUT_SECONDS
-        ) as response, package_path.open("wb") as output:
-            shutil.copyfileobj(response, output)
+        for attempt in range(3):
+            try:
+                with urllib.request.urlopen(
+                    package["url"], timeout=DOWNLOAD_TIMEOUT_SECONDS
+                ) as response, package_path.open("wb") as output:
+                    shutil.copyfileobj(response, output)
+                break
+            except urllib.error.HTTPError as error:
+                if error.code < 500 or attempt == 2:
+                    raise
+                error.close()
+                time.sleep(2**attempt)
     actual = sha256_file(package_path)
     if actual != package["sha256"]:
         raise ValueError(
diff --git a/scripts/test_package_windows_conpty.py b/scripts/test_package_windows_conpty.py
index 0736208b..b92edaf3 100644
--- a/scripts/test_package_windows_conpty.py
+++ b/scripts/test_package_windows_conpty.py
@@ -6,6 +6,7 @@ import json
 import struct
 import tempfile
 import unittest
+import urllib.error
 import zipfile
 from pathlib import Path
 from unittest import mock
@@ -52,7 +53,7 @@ class WindowsConptyPackageTests(unittest.TestCase):
         self.assertIn('conpty\\conpty.dll', wrapper)
         self.assertIn('"*Microsoft Corporation*"', wrapper)
 
-    def test_package_download_has_a_finite_timeout(self) -> None:
+    def test_package_download_retries_server_errors_with_a_finite_timeout(self) -> None:
         payload = b"package"
         with tempfile.TemporaryDirectory() as temporary:
             destination = Path(temporary) / "conpty.nupkg"
@@ -60,14 +61,31 @@ class WindowsConptyPackageTests(unittest.TestCase):
                 "url": "https://example.invalid/conpty.nupkg",
                 "sha256": hashlib.sha256(payload).hexdigest(),
             }
-            with mock.patch.object(
-                package.urllib.request, "urlopen", return_value=io.BytesIO(payload)
-            ) as urlopen:
+            server_error = urllib.error.HTTPError(
+                metadata["url"], 504, "Gateway Time-out", {}, None
+            )
+            with (
+                mock.patch.object(
+                    package.urllib.request,
+                    "urlopen",
+                    side_effect=[server_error, io.BytesIO(payload)],
+                ) as urlopen,
+                mock.patch.object(package.time, "sleep") as sleep,
+            ):
                 package.acquire_package(metadata, destination)
 
-            urlopen.assert_called_once_with(
-                metadata["url"], timeout=package.DOWNLOAD_TIMEOUT_SECONDS
+            self.assertEqual(
+                urlopen.call_args_list,
+                [
+                    mock.call(
+                        metadata["url"], timeout=package.DOWNLOAD_TIMEOUT_SECONDS
+                    ),
+                    mock.call(
+                        metadata["url"], timeout=package.DOWNLOAD_TIMEOUT_SECONDS
+                    ),
+                ],
             )
+            sleep.assert_called_once_with(1)
 
     def test_stage_and_archive_validate_exact_package(self) -> None:
         with tempfile.TemporaryDirectory() as temporary: