Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/pandas/_testing/_io.py: 19%
130 statements
« prev ^ index » next coverage.py v6.4.4, created at 2023-07-17 14:22 -0600
1from __future__ import annotations
3import bz2
4from functools import wraps
5import gzip
6import io
7import socket
8import tarfile
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Callable,
13)
14import zipfile
16from pandas._typing import (
17 FilePath,
18 ReadPickleBuffer,
19)
20from pandas.compat import get_lzma_file
21from pandas.compat._optional import import_optional_dependency
23import pandas as pd
24from pandas._testing._random import rands
25from pandas._testing.contexts import ensure_clean
27from pandas.io.common import urlopen
29if TYPE_CHECKING: 29 ↛ 30line 29 didn't jump to line 30, because the condition on line 29 was never true
30 from pandas import (
31 DataFrame,
32 Series,
33 )
35# skip tests on exceptions with these messages
36_network_error_messages = (
37 # 'urlopen error timed out',
38 # 'timeout: timed out',
39 # 'socket.timeout: timed out',
40 "timed out",
41 "Server Hangup",
42 "HTTP Error 503: Service Unavailable",
43 "502: Proxy Error",
44 "HTTP Error 502: internal error",
45 "HTTP Error 502",
46 "HTTP Error 503",
47 "HTTP Error 403",
48 "HTTP Error 400",
49 "Temporary failure in name resolution",
50 "Name or service not known",
51 "Connection refused",
52 "certificate verify",
53)
55# or this e.errno/e.reason.errno
56_network_errno_vals = (
57 101, # Network is unreachable
58 111, # Connection refused
59 110, # Connection timed out
60 104, # Connection reset Error
61 54, # Connection reset by peer
62 60, # urllib.error.URLError: [Errno 60] Connection timed out
63)
65# Both of the above shouldn't mask real issues such as 404's
66# or refused connections (changed DNS).
67# But some tests (test_data yahoo) contact incredibly flakey
68# servers.
70# and conditionally raise on exception types in _get_default_network_errors
73def _get_default_network_errors():
74 # Lazy import for http.client & urllib.error
75 # because it imports many things from the stdlib
76 import http.client
77 import urllib.error
79 return (
80 OSError,
81 http.client.HTTPException,
82 TimeoutError,
83 urllib.error.URLError,
84 socket.timeout,
85 )
88def optional_args(decorator):
89 """
90 allows a decorator to take optional positional and keyword arguments.
91 Assumes that taking a single, callable, positional argument means that
92 it is decorating a function, i.e. something like this::
94 @my_decorator
95 def function(): pass
97 Calls decorator with decorator(f, *args, **kwargs)
98 """
100 @wraps(decorator)
101 def wrapper(*args, **kwargs):
102 def dec(f):
103 return decorator(f, *args, **kwargs)
105 is_decorating = not kwargs and len(args) == 1 and callable(args[0])
106 if is_decorating:
107 f = args[0]
108 args = ()
109 return dec(f)
110 else:
111 return dec
113 return wrapper
116@optional_args
117def network(
118 t,
119 url="https://www.google.com",
120 raise_on_error=False,
121 check_before_test=False,
122 error_classes=None,
123 skip_errnos=_network_errno_vals,
124 _skip_on_messages=_network_error_messages,
125):
126 """
127 Label a test as requiring network connection and, if an error is
128 encountered, only raise if it does not find a network connection.
130 In comparison to ``network``, this assumes an added contract to your test:
131 you must assert that, under normal conditions, your test will ONLY fail if
132 it does not have network connectivity.
134 You can call this in 3 ways: as a standard decorator, with keyword
135 arguments, or with a positional argument that is the url to check.
137 Parameters
138 ----------
139 t : callable
140 The test requiring network connectivity.
141 url : path
142 The url to test via ``pandas.io.common.urlopen`` to check
143 for connectivity. Defaults to 'https://www.google.com'.
144 raise_on_error : bool
145 If True, never catches errors.
146 check_before_test : bool
147 If True, checks connectivity before running the test case.
148 error_classes : tuple or Exception
149 error classes to ignore. If not in ``error_classes``, raises the error.
150 defaults to OSError. Be careful about changing the error classes here.
151 skip_errnos : iterable of int
152 Any exception that has .errno or .reason.erno set to one
153 of these values will be skipped with an appropriate
154 message.
155 _skip_on_messages: iterable of string
156 any exception e for which one of the strings is
157 a substring of str(e) will be skipped with an appropriate
158 message. Intended to suppress errors where an errno isn't available.
160 Notes
161 -----
162 * ``raise_on_error`` supersedes ``check_before_test``
164 Returns
165 -------
166 t : callable
167 The decorated test ``t``, with checks for connectivity errors.
169 Example
170 -------
172 Tests decorated with @network will fail if it's possible to make a network
173 connection to another URL (defaults to google.com)::
175 >>> from pandas import _testing as tm
176 >>> @tm.network
177 ... def test_network():
178 ... with pd.io.common.urlopen("rabbit://bonanza.com"):
179 ... pass
180 >>> test_network() # doctest: +SKIP
181 Traceback
182 ...
183 URLError: <urlopen error unknown url type: rabbit>
185 You can specify alternative URLs::
187 >>> @tm.network("https://www.yahoo.com")
188 ... def test_something_with_yahoo():
189 ... raise OSError("Failure Message")
190 >>> test_something_with_yahoo() # doctest: +SKIP
191 Traceback (most recent call last):
192 ...
193 OSError: Failure Message
195 If you set check_before_test, it will check the url first and not run the
196 test on failure::
198 >>> @tm.network("failing://url.blaher", check_before_test=True)
199 ... def test_something():
200 ... print("I ran!")
201 ... raise ValueError("Failure")
202 >>> test_something() # doctest: +SKIP
203 Traceback (most recent call last):
204 ...
206 Errors not related to networking will always be raised.
207 """
208 import pytest
210 if error_classes is None:
211 error_classes = _get_default_network_errors()
213 t.network = True
215 @wraps(t)
216 def wrapper(*args, **kwargs):
217 if (
218 check_before_test
219 and not raise_on_error
220 and not can_connect(url, error_classes)
221 ):
222 pytest.skip(
223 f"May not have network connectivity because cannot connect to {url}"
224 )
225 try:
226 return t(*args, **kwargs)
227 except Exception as err:
228 errno = getattr(err, "errno", None)
229 if not errno and hasattr(errno, "reason"):
230 # error: "Exception" has no attribute "reason"
231 errno = getattr(err.reason, "errno", None) # type: ignore[attr-defined]
233 if errno in skip_errnos:
234 pytest.skip(f"Skipping test due to known errno and error {err}")
236 e_str = str(err)
238 if any(m.lower() in e_str.lower() for m in _skip_on_messages):
239 pytest.skip(
240 f"Skipping test because exception message is known and error {err}"
241 )
243 if not isinstance(err, error_classes) or raise_on_error:
244 raise
245 else:
246 pytest.skip(
247 f"Skipping test due to lack of connectivity and error {err}"
248 )
250 return wrapper
253def can_connect(url, error_classes=None) -> bool:
254 """
255 Try to connect to the given url. True if succeeds, False if OSError
256 raised
258 Parameters
259 ----------
260 url : basestring
261 The URL to try to connect to
263 Returns
264 -------
265 connectable : bool
266 Return True if no OSError (unable to connect) or URLError (bad url) was
267 raised
268 """
269 if error_classes is None:
270 error_classes = _get_default_network_errors()
272 try:
273 with urlopen(url, timeout=20) as response:
274 # Timeout just in case rate-limiting is applied
275 if response.status != 200:
276 return False
277 except error_classes:
278 return False
279 else:
280 return True
283# ------------------------------------------------------------------
284# File-IO
287def round_trip_pickle(
288 obj: Any, path: FilePath | ReadPickleBuffer | None = None
289) -> DataFrame | Series:
290 """
291 Pickle an object and then read it again.
293 Parameters
294 ----------
295 obj : any object
296 The object to pickle and then re-read.
297 path : str, path object or file-like object, default None
298 The path where the pickled object is written and then read.
300 Returns
301 -------
302 pandas object
303 The original object that was pickled and then re-read.
304 """
305 _path = path
306 if _path is None:
307 _path = f"__{rands(10)}__.pickle"
308 with ensure_clean(_path) as temp_path:
309 pd.to_pickle(obj, temp_path)
310 return pd.read_pickle(temp_path)
313def round_trip_pathlib(writer, reader, path: str | None = None):
314 """
315 Write an object to file specified by a pathlib.Path and read it back
317 Parameters
318 ----------
319 writer : callable bound to pandas object
320 IO writing function (e.g. DataFrame.to_csv )
321 reader : callable
322 IO reading function (e.g. pd.read_csv )
323 path : str, default None
324 The path where the object is written and then read.
326 Returns
327 -------
328 pandas object
329 The original object that was serialized and then re-read.
330 """
331 import pytest
333 Path = pytest.importorskip("pathlib").Path
334 if path is None:
335 path = "___pathlib___"
336 with ensure_clean(path) as path:
337 writer(Path(path))
338 obj = reader(Path(path))
339 return obj
342def round_trip_localpath(writer, reader, path: str | None = None):
343 """
344 Write an object to file specified by a py.path LocalPath and read it back.
346 Parameters
347 ----------
348 writer : callable bound to pandas object
349 IO writing function (e.g. DataFrame.to_csv )
350 reader : callable
351 IO reading function (e.g. pd.read_csv )
352 path : str, default None
353 The path where the object is written and then read.
355 Returns
356 -------
357 pandas object
358 The original object that was serialized and then re-read.
359 """
360 import pytest
362 LocalPath = pytest.importorskip("py.path").local
363 if path is None:
364 path = "___localpath___"
365 with ensure_clean(path) as path:
366 writer(LocalPath(path))
367 obj = reader(LocalPath(path))
368 return obj
371def write_to_compressed(compression, path, data, dest="test"):
372 """
373 Write data to a compressed file.
375 Parameters
376 ----------
377 compression : {'gzip', 'bz2', 'zip', 'xz', 'zstd'}
378 The compression type to use.
379 path : str
380 The file path to write the data.
381 data : str
382 The data to write.
383 dest : str, default "test"
384 The destination file (for ZIP only)
386 Raises
387 ------
388 ValueError : An invalid compression value was passed in.
389 """
390 args: tuple[Any, ...] = (data,)
391 mode = "wb"
392 method = "write"
393 compress_method: Callable
395 if compression == "zip":
396 compress_method = zipfile.ZipFile
397 mode = "w"
398 args = (dest, data)
399 method = "writestr"
400 elif compression == "tar":
401 compress_method = tarfile.TarFile
402 mode = "w"
403 file = tarfile.TarInfo(name=dest)
404 bytes = io.BytesIO(data)
405 file.size = len(data)
406 args = (file, bytes)
407 method = "addfile"
408 elif compression == "gzip":
409 compress_method = gzip.GzipFile
410 elif compression == "bz2":
411 compress_method = bz2.BZ2File
412 elif compression == "zstd":
413 compress_method = import_optional_dependency("zstandard").open
414 elif compression == "xz":
415 compress_method = get_lzma_file()
416 else:
417 raise ValueError(f"Unrecognized compression type: {compression}")
419 with compress_method(path, mode=mode) as f:
420 getattr(f, method)(*args)
423# ------------------------------------------------------------------
424# Plotting
427def close(fignum=None) -> None:
428 from matplotlib.pyplot import (
429 close as _close,
430 get_fignums,
431 )
433 if fignum is None:
434 for fignum in get_fignums():
435 _close(fignum)
436 else:
437 _close(fignum)