Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/urllib3/util/retry.py: 27%
213 statements
« prev ^ index » next coverage.py v6.4.4, created at 2023-07-17 14:22 -0600
1from __future__ import absolute_import
3import email
4import logging
5import re
6import time
7import warnings
8from collections import namedtuple
9from itertools import takewhile
11from ..exceptions import (
12 ConnectTimeoutError,
13 InvalidHeader,
14 MaxRetryError,
15 ProtocolError,
16 ProxyError,
17 ReadTimeoutError,
18 ResponseError,
19)
20from ..packages import six
22log = logging.getLogger(__name__)
25# Data structure for representing the metadata of requests that result in a retry.
26RequestHistory = namedtuple(
27 "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
28)
31# TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
32_Default = object()
35class _RetryMeta(type):
36 @property
37 def DEFAULT_METHOD_WHITELIST(cls):
38 warnings.warn(
39 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
40 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
41 DeprecationWarning,
42 )
43 return cls.DEFAULT_ALLOWED_METHODS
45 @DEFAULT_METHOD_WHITELIST.setter
46 def DEFAULT_METHOD_WHITELIST(cls, value):
47 warnings.warn(
48 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
49 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
50 DeprecationWarning,
51 )
52 cls.DEFAULT_ALLOWED_METHODS = value
54 @property
55 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
56 warnings.warn(
57 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
58 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
59 DeprecationWarning,
60 )
61 return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
63 @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
64 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
65 warnings.warn(
66 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
67 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
68 DeprecationWarning,
69 )
70 cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
72 @property
73 def BACKOFF_MAX(cls):
74 warnings.warn(
75 "Using 'Retry.BACKOFF_MAX' is deprecated and "
76 "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
77 DeprecationWarning,
78 )
79 return cls.DEFAULT_BACKOFF_MAX
81 @BACKOFF_MAX.setter
82 def BACKOFF_MAX(cls, value):
83 warnings.warn(
84 "Using 'Retry.BACKOFF_MAX' is deprecated and "
85 "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
86 DeprecationWarning,
87 )
88 cls.DEFAULT_BACKOFF_MAX = value
91@six.add_metaclass(_RetryMeta)
92class Retry(object):
93 """Retry configuration.
95 Each retry attempt will create a new Retry object with updated values, so
96 they can be safely reused.
98 Retries can be defined as a default for a pool::
100 retries = Retry(connect=5, read=2, redirect=5)
101 http = PoolManager(retries=retries)
102 response = http.request('GET', 'http://example.com/')
104 Or per-request (which overrides the default for the pool)::
106 response = http.request('GET', 'http://example.com/', retries=Retry(10))
108 Retries can be disabled by passing ``False``::
110 response = http.request('GET', 'http://example.com/', retries=False)
112 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
113 retries are disabled, in which case the causing exception will be raised.
115 :param int total:
116 Total number of retries to allow. Takes precedence over other counts.
118 Set to ``None`` to remove this constraint and fall back on other
119 counts.
121 Set to ``0`` to fail on the first retry.
123 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
125 :param int connect:
126 How many connection-related errors to retry on.
128 These are errors raised before the request is sent to the remote server,
129 which we assume has not triggered the server to process the request.
131 Set to ``0`` to fail on the first retry of this type.
133 :param int read:
134 How many times to retry on read errors.
136 These errors are raised after the request was sent to the server, so the
137 request may have side-effects.
139 Set to ``0`` to fail on the first retry of this type.
141 :param int redirect:
142 How many redirects to perform. Limit this to avoid infinite redirect
143 loops.
145 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
146 308.
148 Set to ``0`` to fail on the first retry of this type.
150 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
152 :param int status:
153 How many times to retry on bad status codes.
155 These are retries made on responses, where status code matches
156 ``status_forcelist``.
158 Set to ``0`` to fail on the first retry of this type.
160 :param int other:
161 How many times to retry on other errors.
163 Other errors are errors that are not connect, read, redirect or status errors.
164 These errors might be raised after the request was sent to the server, so the
165 request might have side-effects.
167 Set to ``0`` to fail on the first retry of this type.
169 If ``total`` is not set, it's a good idea to set this to 0 to account
170 for unexpected edge cases and avoid infinite retry loops.
172 :param iterable allowed_methods:
173 Set of uppercased HTTP method verbs that we should retry on.
175 By default, we only retry on methods which are considered to be
176 idempotent (multiple requests with the same parameters end with the
177 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
179 Set to a ``False`` value to retry on any verb.
181 .. warning::
183 Previously this parameter was named ``method_whitelist``, that
184 usage is deprecated in v1.26.0 and will be removed in v2.0.
186 :param iterable status_forcelist:
187 A set of integer HTTP status codes that we should force a retry on.
188 A retry is initiated if the request method is in ``allowed_methods``
189 and the response status code is in ``status_forcelist``.
191 By default, this is disabled with ``None``.
193 :param float backoff_factor:
194 A backoff factor to apply between attempts after the second try
195 (most errors are resolved immediately by a second try without a
196 delay). urllib3 will sleep for::
198 {backoff factor} * (2 ** ({number of total retries} - 1))
200 seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
201 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
202 than :attr:`Retry.DEFAULT_BACKOFF_MAX`.
204 By default, backoff is disabled (set to 0).
206 :param bool raise_on_redirect: Whether, if the number of redirects is
207 exhausted, to raise a MaxRetryError, or to return a response with a
208 response code in the 3xx range.
210 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
211 whether we should raise an exception, or return a response,
212 if status falls in ``status_forcelist`` range and retries have
213 been exhausted.
215 :param tuple history: The history of the request encountered during
216 each call to :meth:`~Retry.increment`. The list is in the order
217 the requests occurred. Each list item is of class :class:`RequestHistory`.
219 :param bool respect_retry_after_header:
220 Whether to respect Retry-After header on status codes defined as
221 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
223 :param iterable remove_headers_on_redirect:
224 Sequence of headers to remove from the request when a response
225 indicating a redirect is returned before firing off the redirected
226 request.
227 """
229 #: Default methods to be used for ``allowed_methods``
230 DEFAULT_ALLOWED_METHODS = frozenset(
231 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
232 )
234 #: Default status codes to be used for ``status_forcelist``
235 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
237 #: Default headers to be used for ``remove_headers_on_redirect``
238 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
240 #: Maximum backoff time.
241 DEFAULT_BACKOFF_MAX = 120
243 def __init__(
244 self,
245 total=10,
246 connect=None,
247 read=None,
248 redirect=None,
249 status=None,
250 other=None,
251 allowed_methods=_Default,
252 status_forcelist=None,
253 backoff_factor=0,
254 raise_on_redirect=True,
255 raise_on_status=True,
256 history=None,
257 respect_retry_after_header=True,
258 remove_headers_on_redirect=_Default,
259 # TODO: Deprecated, remove in v2.0
260 method_whitelist=_Default,
261 ):
263 if method_whitelist is not _Default: 263 ↛ 264line 263 didn't jump to line 264, because the condition on line 263 was never true
264 if allowed_methods is not _Default:
265 raise ValueError(
266 "Using both 'allowed_methods' and "
267 "'method_whitelist' together is not allowed. "
268 "Instead only use 'allowed_methods'"
269 )
270 warnings.warn(
271 "Using 'method_whitelist' with Retry is deprecated and "
272 "will be removed in v2.0. Use 'allowed_methods' instead",
273 DeprecationWarning,
274 stacklevel=2,
275 )
276 allowed_methods = method_whitelist
277 if allowed_methods is _Default: 277 ↛ 279line 277 didn't jump to line 279, because the condition on line 277 was never false
278 allowed_methods = self.DEFAULT_ALLOWED_METHODS
279 if remove_headers_on_redirect is _Default: 279 ↛ 282line 279 didn't jump to line 282, because the condition on line 279 was never false
280 remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
282 self.total = total
283 self.connect = connect
284 self.read = read
285 self.status = status
286 self.other = other
288 if redirect is False or total is False: 288 ↛ 289line 288 didn't jump to line 289, because the condition on line 288 was never true
289 redirect = 0
290 raise_on_redirect = False
292 self.redirect = redirect
293 self.status_forcelist = status_forcelist or set()
294 self.allowed_methods = allowed_methods
295 self.backoff_factor = backoff_factor
296 self.raise_on_redirect = raise_on_redirect
297 self.raise_on_status = raise_on_status
298 self.history = history or tuple()
299 self.respect_retry_after_header = respect_retry_after_header
300 self.remove_headers_on_redirect = frozenset(
301 [h.lower() for h in remove_headers_on_redirect]
302 )
304 def new(self, **kw):
305 params = dict(
306 total=self.total,
307 connect=self.connect,
308 read=self.read,
309 redirect=self.redirect,
310 status=self.status,
311 other=self.other,
312 status_forcelist=self.status_forcelist,
313 backoff_factor=self.backoff_factor,
314 raise_on_redirect=self.raise_on_redirect,
315 raise_on_status=self.raise_on_status,
316 history=self.history,
317 remove_headers_on_redirect=self.remove_headers_on_redirect,
318 respect_retry_after_header=self.respect_retry_after_header,
319 )
321 # TODO: If already given in **kw we use what's given to us
322 # If not given we need to figure out what to pass. We decide
323 # based on whether our class has the 'method_whitelist' property
324 # and if so we pass the deprecated 'method_whitelist' otherwise
325 # we use 'allowed_methods'. Remove in v2.0
326 if "method_whitelist" not in kw and "allowed_methods" not in kw:
327 if "method_whitelist" in self.__dict__:
328 warnings.warn(
329 "Using 'method_whitelist' with Retry is deprecated and "
330 "will be removed in v2.0. Use 'allowed_methods' instead",
331 DeprecationWarning,
332 )
333 params["method_whitelist"] = self.allowed_methods
334 else:
335 params["allowed_methods"] = self.allowed_methods
337 params.update(kw)
338 return type(self)(**params)
340 @classmethod
341 def from_int(cls, retries, redirect=True, default=None):
342 """Backwards-compatibility for the old retries format."""
343 if retries is None:
344 retries = default if default is not None else cls.DEFAULT
346 if isinstance(retries, Retry):
347 return retries
349 redirect = bool(redirect) and None
350 new_retries = cls(retries, redirect=redirect)
351 log.debug("Converted retries value: %r -> %r", retries, new_retries)
352 return new_retries
354 def get_backoff_time(self):
355 """Formula for computing the current backoff
357 :rtype: float
358 """
359 # We want to consider only the last consecutive errors sequence (Ignore redirects).
360 consecutive_errors_len = len(
361 list(
362 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
363 )
364 )
365 if consecutive_errors_len <= 1:
366 return 0
368 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
369 return min(self.DEFAULT_BACKOFF_MAX, backoff_value)
371 def parse_retry_after(self, retry_after):
372 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
373 if re.match(r"^\s*[0-9]+\s*$", retry_after):
374 seconds = int(retry_after)
375 else:
376 retry_date_tuple = email.utils.parsedate_tz(retry_after)
377 if retry_date_tuple is None:
378 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
379 if retry_date_tuple[9] is None: # Python 2
380 # Assume UTC if no timezone was specified
381 # On Python2.7, parsedate_tz returns None for a timezone offset
382 # instead of 0 if no timezone is given, where mktime_tz treats
383 # a None timezone offset as local time.
384 retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
386 retry_date = email.utils.mktime_tz(retry_date_tuple)
387 seconds = retry_date - time.time()
389 if seconds < 0:
390 seconds = 0
392 return seconds
394 def get_retry_after(self, response):
395 """Get the value of Retry-After in seconds."""
397 retry_after = response.headers.get("Retry-After")
399 if retry_after is None:
400 return None
402 return self.parse_retry_after(retry_after)
404 def sleep_for_retry(self, response=None):
405 retry_after = self.get_retry_after(response)
406 if retry_after:
407 time.sleep(retry_after)
408 return True
410 return False
412 def _sleep_backoff(self):
413 backoff = self.get_backoff_time()
414 if backoff <= 0:
415 return
416 time.sleep(backoff)
418 def sleep(self, response=None):
419 """Sleep between retry attempts.
421 This method will respect a server's ``Retry-After`` response header
422 and sleep the duration of the time requested. If that is not present, it
423 will use an exponential backoff. By default, the backoff factor is 0 and
424 this method will return immediately.
425 """
427 if self.respect_retry_after_header and response:
428 slept = self.sleep_for_retry(response)
429 if slept:
430 return
432 self._sleep_backoff()
434 def _is_connection_error(self, err):
435 """Errors when we're fairly sure that the server did not receive the
436 request, so it should be safe to retry.
437 """
438 if isinstance(err, ProxyError):
439 err = err.original_error
440 return isinstance(err, ConnectTimeoutError)
442 def _is_read_error(self, err):
443 """Errors that occur after the request has been started, so we should
444 assume that the server began processing it.
445 """
446 return isinstance(err, (ReadTimeoutError, ProtocolError))
448 def _is_method_retryable(self, method):
449 """Checks if a given HTTP method should be retried upon, depending if
450 it is included in the allowed_methods
451 """
452 # TODO: For now favor if the Retry implementation sets its own method_whitelist
453 # property outside of our constructor to avoid breaking custom implementations.
454 if "method_whitelist" in self.__dict__:
455 warnings.warn(
456 "Using 'method_whitelist' with Retry is deprecated and "
457 "will be removed in v2.0. Use 'allowed_methods' instead",
458 DeprecationWarning,
459 )
460 allowed_methods = self.method_whitelist
461 else:
462 allowed_methods = self.allowed_methods
464 if allowed_methods and method.upper() not in allowed_methods:
465 return False
466 return True
468 def is_retry(self, method, status_code, has_retry_after=False):
469 """Is this method/status code retryable? (Based on allowlists and control
470 variables such as the number of total retries to allow, whether to
471 respect the Retry-After header, whether this header is present, and
472 whether the returned status code is on the list of status codes to
473 be retried upon on the presence of the aforementioned header)
474 """
475 if not self._is_method_retryable(method):
476 return False
478 if self.status_forcelist and status_code in self.status_forcelist:
479 return True
481 return (
482 self.total
483 and self.respect_retry_after_header
484 and has_retry_after
485 and (status_code in self.RETRY_AFTER_STATUS_CODES)
486 )
488 def is_exhausted(self):
489 """Are we out of retries?"""
490 retry_counts = (
491 self.total,
492 self.connect,
493 self.read,
494 self.redirect,
495 self.status,
496 self.other,
497 )
498 retry_counts = list(filter(None, retry_counts))
499 if not retry_counts:
500 return False
502 return min(retry_counts) < 0
504 def increment(
505 self,
506 method=None,
507 url=None,
508 response=None,
509 error=None,
510 _pool=None,
511 _stacktrace=None,
512 ):
513 """Return a new Retry object with incremented retry counters.
515 :param response: A response object, or None, if the server did not
516 return a response.
517 :type response: :class:`~urllib3.response.HTTPResponse`
518 :param Exception error: An error encountered during the request, or
519 None if the response was received successfully.
521 :return: A new ``Retry`` object.
522 """
523 if self.total is False and error:
524 # Disabled, indicate to re-raise the error.
525 raise six.reraise(type(error), error, _stacktrace)
527 total = self.total
528 if total is not None:
529 total -= 1
531 connect = self.connect
532 read = self.read
533 redirect = self.redirect
534 status_count = self.status
535 other = self.other
536 cause = "unknown"
537 status = None
538 redirect_location = None
540 if error and self._is_connection_error(error):
541 # Connect retry?
542 if connect is False:
543 raise six.reraise(type(error), error, _stacktrace)
544 elif connect is not None:
545 connect -= 1
547 elif error and self._is_read_error(error):
548 # Read retry?
549 if read is False or not self._is_method_retryable(method):
550 raise six.reraise(type(error), error, _stacktrace)
551 elif read is not None:
552 read -= 1
554 elif error:
555 # Other retry?
556 if other is not None:
557 other -= 1
559 elif response and response.get_redirect_location():
560 # Redirect retry?
561 if redirect is not None:
562 redirect -= 1
563 cause = "too many redirects"
564 redirect_location = response.get_redirect_location()
565 status = response.status
567 else:
568 # Incrementing because of a server error like a 500 in
569 # status_forcelist and the given method is in the allowed_methods
570 cause = ResponseError.GENERIC_ERROR
571 if response and response.status:
572 if status_count is not None:
573 status_count -= 1
574 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
575 status = response.status
577 history = self.history + (
578 RequestHistory(method, url, error, status, redirect_location),
579 )
581 new_retry = self.new(
582 total=total,
583 connect=connect,
584 read=read,
585 redirect=redirect,
586 status=status_count,
587 other=other,
588 history=history,
589 )
591 if new_retry.is_exhausted():
592 raise MaxRetryError(_pool, url, error or ResponseError(cause))
594 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
596 return new_retry
598 def __repr__(self):
599 return (
600 "{cls.__name__}(total={self.total}, connect={self.connect}, "
601 "read={self.read}, redirect={self.redirect}, status={self.status})"
602 ).format(cls=type(self), self=self)
604 def __getattr__(self, item):
605 if item == "method_whitelist":
606 # TODO: Remove this deprecated alias in v2.0
607 warnings.warn(
608 "Using 'method_whitelist' with Retry is deprecated and "
609 "will be removed in v2.0. Use 'allowed_methods' instead",
610 DeprecationWarning,
611 )
612 return self.allowed_methods
613 try:
614 return getattr(super(Retry, self), item)
615 except AttributeError:
616 return getattr(Retry, item)
619# For backwards compatibility (equivalent to pre-v1.9):
620Retry.DEFAULT = Retry(3)