Skip to content

Commit 3bbdfe5

Browse files
Support urllib3 1.26 and test that elapsed time counts toward the key lifetime
Import BaseHTTPResponse only for type checking and take the next backoff from get_backoff_time(), since no jitter is configured. Co-authored-by: Rahul Shivu Mahadev <51690557+rahulsmahadev@users.noreply.github.com>
1 parent c4afc68 commit 3bbdfe5

2 files changed

Lines changed: 35 additions & 3 deletions

File tree

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
from typing_extensions import override
4343
from urllib3.connectionpool import ConnectionPool
4444
from urllib3.exceptions import MaxRetryError, ResponseError
45-
from urllib3.response import BaseHTTPResponse
4645
from urllib3.util.retry import Retry
4746

4847
from pyiceberg import __version__
@@ -122,6 +121,7 @@
122121

123122
if TYPE_CHECKING:
124123
import pyarrow as pa
124+
from urllib3.response import BaseHTTPResponse
125125

126126
logger = logging.getLogger(__name__)
127127

@@ -564,8 +564,8 @@ def _max_sleep(self, response: BaseHTTPResponse | None) -> float:
564564
if self.respect_retry_after_header and response is not None:
565565
if retry_after := self.get_retry_after(response):
566566
return retry_after
567-
# `get_backoff_time` adds a random jitter in [0, backoff_jitter), so take the worst case.
568-
return min(self.backoff_max, self.new(backoff_jitter=0.0).get_backoff_time() + self.backoff_jitter)
567+
# No jitter is configured, so this is exactly what the next backoff will be.
568+
return self.get_backoff_time()
569569

570570

571571
class _RetryTimeoutHTTPAdapter(HTTPAdapter):

‎tests/catalog/test_rest.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import os
2222
import pickle
2323
import platform
24+
import time
2425
import uuid
2526
from collections.abc import Callable, Iterator
2627
from contextlib import contextmanager
@@ -2715,6 +2716,37 @@ def test_session_does_not_retry_keyed_post_past_the_key_lifetime_due_to_backoff(
27152716
assert server["namespace_calls"] == 2
27162717

27172718

2719+
def test_session_counts_time_already_spent_toward_the_key_lifetime() -> None:
2720+
# Each Retry-After (4s) alone fits in the lifetime (PT10S), but after two of them (8s) the third would not.
2721+
# A fake clock stands in for the real one so the test does not wait out the sleeps.
2722+
clock = {"now": 1000.0}
2723+
2724+
def fake_sleep(seconds: float) -> None:
2725+
clock["now"] += seconds
2726+
2727+
fake_time = mock.Mock(wraps=time)
2728+
fake_time.monotonic.side_effect = lambda: clock["now"]
2729+
fake_time.sleep.side_effect = fake_sleep
2730+
2731+
with (
2732+
_local_rest_server_503_then_200(num_failures=3, idempotency_key_lifetime="PT10S", retry_after="4") as server,
2733+
mock.patch("pyiceberg.catalog.rest.time", fake_time),
2734+
mock.patch("urllib3.util.retry.time", fake_time),
2735+
):
2736+
catalog = RestCatalog(
2737+
"rest",
2738+
**{
2739+
"uri": f"http://127.0.0.1:{server['port']}/",
2740+
"token": TEST_TOKEN,
2741+
REST_CLIENT_MAX_RETRIES: "3",
2742+
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
2743+
},
2744+
)
2745+
with pytest.raises(ServiceUnavailableError):
2746+
catalog.create_namespace("foo")
2747+
assert server["namespace_calls"] == 3
2748+
2749+
27182750
def test_session_retries_keyed_post_within_the_key_lifetime_with_retry_after() -> None:
27192751
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT30M", retry_after="0") as server:
27202752
catalog = RestCatalog(

0 commit comments

Comments
 (0)