Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/pandas/core/reshape/concat.py: 10%
278 statements
« prev ^ index » next coverage.py v6.4.4, created at 2023-07-17 14:22 -0600
1"""
2Concat routines.
3"""
4from __future__ import annotations
6from collections import abc
7from typing import (
8 TYPE_CHECKING,
9 Callable,
10 Hashable,
11 Iterable,
12 Literal,
13 Mapping,
14 cast,
15 overload,
16)
17import warnings
19import numpy as np
21from pandas._typing import (
22 Axis,
23 HashableT,
24)
25from pandas.util._decorators import (
26 cache_readonly,
27 deprecate_nonkeyword_arguments,
28)
29from pandas.util._exceptions import find_stack_level
31from pandas.core.dtypes.concat import concat_compat
32from pandas.core.dtypes.generic import (
33 ABCDataFrame,
34 ABCSeries,
35)
36from pandas.core.dtypes.inference import is_bool
37from pandas.core.dtypes.missing import isna
39from pandas.core.arrays.categorical import (
40 factorize_from_iterable,
41 factorize_from_iterables,
42)
43import pandas.core.common as com
44from pandas.core.indexes.api import (
45 Index,
46 MultiIndex,
47 all_indexes_same,
48 default_index,
49 ensure_index,
50 get_objs_combined_axis,
51 get_unanimous_names,
52)
53from pandas.core.internals import concatenate_managers
55if TYPE_CHECKING: 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true
56 from pandas import (
57 DataFrame,
58 Series,
59 )
60 from pandas.core.generic import NDFrame
62# ---------------------------------------------------------------------
63# Concatenate DataFrame objects
66@overload
67def concat(
68 objs: Iterable[DataFrame] | Mapping[HashableT, DataFrame],
69 axis: Literal[0, "index"] = ...,
70 join: str = ...,
71 ignore_index: bool = ...,
72 keys=...,
73 levels=...,
74 names=...,
75 verify_integrity: bool = ...,
76 sort: bool = ...,
77 copy: bool = ...,
78) -> DataFrame:
79 ...
82@overload
83def concat(
84 objs: Iterable[Series] | Mapping[HashableT, Series],
85 axis: Literal[0, "index"] = ...,
86 join: str = ...,
87 ignore_index: bool = ...,
88 keys=...,
89 levels=...,
90 names=...,
91 verify_integrity: bool = ...,
92 sort: bool = ...,
93 copy: bool = ...,
94) -> Series:
95 ...
98@overload
99def concat(
100 objs: Iterable[NDFrame] | Mapping[HashableT, NDFrame],
101 axis: Literal[0, "index"] = ...,
102 join: str = ...,
103 ignore_index: bool = ...,
104 keys=...,
105 levels=...,
106 names=...,
107 verify_integrity: bool = ...,
108 sort: bool = ...,
109 copy: bool = ...,
110) -> DataFrame | Series:
111 ...
114@overload
115def concat(
116 objs: Iterable[NDFrame] | Mapping[HashableT, NDFrame],
117 axis: Literal[1, "columns"],
118 join: str = ...,
119 ignore_index: bool = ...,
120 keys=...,
121 levels=...,
122 names=...,
123 verify_integrity: bool = ...,
124 sort: bool = ...,
125 copy: bool = ...,
126) -> DataFrame:
127 ...
130@overload
131def concat(
132 objs: Iterable[NDFrame] | Mapping[HashableT, NDFrame],
133 axis: Axis = ...,
134 join: str = ...,
135 ignore_index: bool = ...,
136 keys=...,
137 levels=...,
138 names=...,
139 verify_integrity: bool = ...,
140 sort: bool = ...,
141 copy: bool = ...,
142) -> DataFrame | Series:
143 ...
146@deprecate_nonkeyword_arguments(version=None, allowed_args=["objs"])
147def concat(
148 objs: Iterable[NDFrame] | Mapping[HashableT, NDFrame],
149 axis: Axis = 0,
150 join: str = "outer",
151 ignore_index: bool = False,
152 keys=None,
153 levels=None,
154 names=None,
155 verify_integrity: bool = False,
156 sort: bool = False,
157 copy: bool = True,
158) -> DataFrame | Series:
159 """
160 Concatenate pandas objects along a particular axis.
162 Allows optional set logic along the other axes.
164 Can also add a layer of hierarchical indexing on the concatenation axis,
165 which may be useful if the labels are the same (or overlapping) on
166 the passed axis number.
168 Parameters
169 ----------
170 objs : a sequence or mapping of Series or DataFrame objects
171 If a mapping is passed, the sorted keys will be used as the `keys`
172 argument, unless it is passed, in which case the values will be
173 selected (see below). Any None objects will be dropped silently unless
174 they are all None in which case a ValueError will be raised.
175 axis : {0/'index', 1/'columns'}, default 0
176 The axis to concatenate along.
177 join : {'inner', 'outer'}, default 'outer'
178 How to handle indexes on other axis (or axes).
179 ignore_index : bool, default False
180 If True, do not use the index values along the concatenation axis. The
181 resulting axis will be labeled 0, ..., n - 1. This is useful if you are
182 concatenating objects where the concatenation axis does not have
183 meaningful indexing information. Note the index values on the other
184 axes are still respected in the join.
185 keys : sequence, default None
186 If multiple levels passed, should contain tuples. Construct
187 hierarchical index using the passed keys as the outermost level.
188 levels : list of sequences, default None
189 Specific levels (unique values) to use for constructing a
190 MultiIndex. Otherwise they will be inferred from the keys.
191 names : list, default None
192 Names for the levels in the resulting hierarchical index.
193 verify_integrity : bool, default False
194 Check whether the new concatenated axis contains duplicates. This can
195 be very expensive relative to the actual data concatenation.
196 sort : bool, default False
197 Sort non-concatenation axis if it is not already aligned when `join`
198 is 'outer'.
199 This has no effect when ``join='inner'``, which already preserves
200 the order of the non-concatenation axis.
202 .. versionchanged:: 1.0.0
204 Changed to not sort by default.
206 copy : bool, default True
207 If False, do not copy data unnecessarily.
209 Returns
210 -------
211 object, type of objs
212 When concatenating all ``Series`` along the index (axis=0), a
213 ``Series`` is returned. When ``objs`` contains at least one
214 ``DataFrame``, a ``DataFrame`` is returned. When concatenating along
215 the columns (axis=1), a ``DataFrame`` is returned.
217 See Also
218 --------
219 DataFrame.join : Join DataFrames using indexes.
220 DataFrame.merge : Merge DataFrames by indexes or columns.
222 Notes
223 -----
224 The keys, levels, and names arguments are all optional.
226 A walkthrough of how this method fits in with other tools for combining
227 pandas objects can be found `here
228 <https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html>`__.
230 It is not recommended to build DataFrames by adding single rows in a
231 for loop. Build a list of rows and make a DataFrame in a single concat.
233 Examples
234 --------
235 Combine two ``Series``.
237 >>> s1 = pd.Series(['a', 'b'])
238 >>> s2 = pd.Series(['c', 'd'])
239 >>> pd.concat([s1, s2])
240 0 a
241 1 b
242 0 c
243 1 d
244 dtype: object
246 Clear the existing index and reset it in the result
247 by setting the ``ignore_index`` option to ``True``.
249 >>> pd.concat([s1, s2], ignore_index=True)
250 0 a
251 1 b
252 2 c
253 3 d
254 dtype: object
256 Add a hierarchical index at the outermost level of
257 the data with the ``keys`` option.
259 >>> pd.concat([s1, s2], keys=['s1', 's2'])
260 s1 0 a
261 1 b
262 s2 0 c
263 1 d
264 dtype: object
266 Label the index keys you create with the ``names`` option.
268 >>> pd.concat([s1, s2], keys=['s1', 's2'],
269 ... names=['Series name', 'Row ID'])
270 Series name Row ID
271 s1 0 a
272 1 b
273 s2 0 c
274 1 d
275 dtype: object
277 Combine two ``DataFrame`` objects with identical columns.
279 >>> df1 = pd.DataFrame([['a', 1], ['b', 2]],
280 ... columns=['letter', 'number'])
281 >>> df1
282 letter number
283 0 a 1
284 1 b 2
285 >>> df2 = pd.DataFrame([['c', 3], ['d', 4]],
286 ... columns=['letter', 'number'])
287 >>> df2
288 letter number
289 0 c 3
290 1 d 4
291 >>> pd.concat([df1, df2])
292 letter number
293 0 a 1
294 1 b 2
295 0 c 3
296 1 d 4
298 Combine ``DataFrame`` objects with overlapping columns
299 and return everything. Columns outside the intersection will
300 be filled with ``NaN`` values.
302 >>> df3 = pd.DataFrame([['c', 3, 'cat'], ['d', 4, 'dog']],
303 ... columns=['letter', 'number', 'animal'])
304 >>> df3
305 letter number animal
306 0 c 3 cat
307 1 d 4 dog
308 >>> pd.concat([df1, df3], sort=False)
309 letter number animal
310 0 a 1 NaN
311 1 b 2 NaN
312 0 c 3 cat
313 1 d 4 dog
315 Combine ``DataFrame`` objects with overlapping columns
316 and return only those that are shared by passing ``inner`` to
317 the ``join`` keyword argument.
319 >>> pd.concat([df1, df3], join="inner")
320 letter number
321 0 a 1
322 1 b 2
323 0 c 3
324 1 d 4
326 Combine ``DataFrame`` objects horizontally along the x axis by
327 passing in ``axis=1``.
329 >>> df4 = pd.DataFrame([['bird', 'polly'], ['monkey', 'george']],
330 ... columns=['animal', 'name'])
331 >>> pd.concat([df1, df4], axis=1)
332 letter number animal name
333 0 a 1 bird polly
334 1 b 2 monkey george
336 Prevent the result from including duplicate index values with the
337 ``verify_integrity`` option.
339 >>> df5 = pd.DataFrame([1], index=['a'])
340 >>> df5
341 0
342 a 1
343 >>> df6 = pd.DataFrame([2], index=['a'])
344 >>> df6
345 0
346 a 2
347 >>> pd.concat([df5, df6], verify_integrity=True)
348 Traceback (most recent call last):
349 ...
350 ValueError: Indexes have overlapping values: ['a']
352 Append a single row to the end of a ``DataFrame`` object.
354 >>> df7 = pd.DataFrame({'a': 1, 'b': 2}, index=[0])
355 >>> df7
356 a b
357 0 1 2
358 >>> new_row = pd.Series({'a': 3, 'b': 4})
359 >>> new_row
360 a 3
361 b 4
362 dtype: int64
363 >>> pd.concat([df7, new_row.to_frame().T], ignore_index=True)
364 a b
365 0 1 2
366 1 3 4
367 """
368 op = _Concatenator(
369 objs,
370 axis=axis,
371 ignore_index=ignore_index,
372 join=join,
373 keys=keys,
374 levels=levels,
375 names=names,
376 verify_integrity=verify_integrity,
377 copy=copy,
378 sort=sort,
379 )
381 return op.get_result()
384class _Concatenator:
385 """
386 Orchestrates a concatenation operation for BlockManagers
387 """
389 def __init__(
390 self,
391 objs: Iterable[NDFrame] | Mapping[HashableT, NDFrame],
392 axis=0,
393 join: str = "outer",
394 keys=None,
395 levels=None,
396 names=None,
397 ignore_index: bool = False,
398 verify_integrity: bool = False,
399 copy: bool = True,
400 sort=False,
401 ) -> None:
402 if isinstance(objs, (ABCSeries, ABCDataFrame, str)):
403 raise TypeError(
404 "first argument must be an iterable of pandas "
405 f'objects, you passed an object of type "{type(objs).__name__}"'
406 )
408 if join == "outer":
409 self.intersect = False
410 elif join == "inner":
411 self.intersect = True
412 else: # pragma: no cover
413 raise ValueError(
414 "Only can inner (intersect) or outer (union) join the other axis"
415 )
417 if isinstance(objs, abc.Mapping):
418 if keys is None:
419 keys = list(objs.keys())
420 objs = [objs[k] for k in keys]
421 else:
422 objs = list(objs)
424 if len(objs) == 0:
425 raise ValueError("No objects to concatenate")
427 if keys is None:
428 objs = list(com.not_none(*objs))
429 else:
430 # #1649
431 clean_keys = []
432 clean_objs = []
433 for k, v in zip(keys, objs):
434 if v is None:
435 continue
436 clean_keys.append(k)
437 clean_objs.append(v)
438 objs = clean_objs
440 if isinstance(keys, MultiIndex):
441 # TODO: retain levels?
442 keys = type(keys).from_tuples(clean_keys, names=keys.names)
443 else:
444 name = getattr(keys, "name", None)
445 keys = Index(clean_keys, name=name)
447 if len(objs) == 0:
448 raise ValueError("All objects passed were None")
450 # figure out what our result ndim is going to be
451 ndims = set()
452 for obj in objs:
453 if not isinstance(obj, (ABCSeries, ABCDataFrame)):
454 msg = (
455 f"cannot concatenate object of type '{type(obj)}'; "
456 "only Series and DataFrame objs are valid"
457 )
458 raise TypeError(msg)
460 ndims.add(obj.ndim)
462 # get the sample
463 # want the highest ndim that we have, and must be non-empty
464 # unless all objs are empty
465 sample: NDFrame | None = None
466 if len(ndims) > 1:
467 max_ndim = max(ndims)
468 for obj in objs:
469 if obj.ndim == max_ndim and np.sum(obj.shape):
470 sample = obj
471 break
473 else:
474 # filter out the empties if we have not multi-index possibilities
475 # note to keep empty Series as it affect to result columns / name
476 non_empties = [
477 obj for obj in objs if sum(obj.shape) > 0 or isinstance(obj, ABCSeries)
478 ]
480 if len(non_empties) and (
481 keys is None and names is None and levels is None and not self.intersect
482 ):
483 objs = non_empties
484 sample = objs[0]
486 if sample is None:
487 sample = objs[0]
488 self.objs = objs
490 # Standardize axis parameter to int
491 if isinstance(sample, ABCSeries):
492 from pandas import DataFrame
494 axis = DataFrame._get_axis_number(axis)
495 else:
496 axis = sample._get_axis_number(axis)
498 # Need to flip BlockManager axis in the DataFrame special case
499 self._is_frame = isinstance(sample, ABCDataFrame)
500 if self._is_frame:
501 axis = sample._get_block_manager_axis(axis)
503 self._is_series = isinstance(sample, ABCSeries)
504 if not 0 <= axis <= sample.ndim:
505 raise AssertionError(
506 f"axis must be between 0 and {sample.ndim}, input was {axis}"
507 )
509 # if we have mixed ndims, then convert to highest ndim
510 # creating column numbers as needed
511 if len(ndims) > 1:
512 current_column = 0
513 max_ndim = sample.ndim
514 self.objs, objs = [], self.objs
515 for obj in objs:
517 ndim = obj.ndim
518 if ndim == max_ndim:
519 pass
521 elif ndim != max_ndim - 1:
522 raise ValueError(
523 "cannot concatenate unaligned mixed "
524 "dimensional NDFrame objects"
525 )
527 else:
528 name = getattr(obj, "name", None)
529 if ignore_index or name is None:
530 name = current_column
531 current_column += 1
533 # doing a row-wise concatenation so need everything
534 # to line up
535 if self._is_frame and axis == 1:
536 name = 0
537 # mypy needs to know sample is not an NDFrame
538 sample = cast("DataFrame | Series", sample)
539 obj = sample._constructor({name: obj})
541 self.objs.append(obj)
543 # note: this is the BlockManager axis (since DataFrame is transposed)
544 self.bm_axis = axis
545 self.axis = 1 - self.bm_axis if self._is_frame else 0
546 self.keys = keys
547 self.names = names or getattr(keys, "names", None)
548 self.levels = levels
550 if not is_bool(sort):
551 warnings.warn(
552 "Passing non boolean values for sort is deprecated and "
553 "will error in a future version!",
554 FutureWarning,
555 stacklevel=find_stack_level(),
556 )
557 self.sort = sort
559 self.ignore_index = ignore_index
560 self.verify_integrity = verify_integrity
561 self.copy = copy
563 self.new_axes = self._get_new_axes()
565 def get_result(self):
566 cons: Callable[..., DataFrame | Series]
567 sample: DataFrame | Series
569 # series only
570 if self._is_series:
571 sample = cast("Series", self.objs[0])
573 # stack blocks
574 if self.bm_axis == 0:
575 name = com.consensus_name_attr(self.objs)
576 cons = sample._constructor
578 arrs = [ser._values for ser in self.objs]
580 res = concat_compat(arrs, axis=0)
581 result = cons(res, index=self.new_axes[0], name=name, dtype=res.dtype)
582 return result.__finalize__(self, method="concat")
584 # combine as columns in a frame
585 else:
586 data = dict(zip(range(len(self.objs)), self.objs))
588 # GH28330 Preserves subclassed objects through concat
589 cons = sample._constructor_expanddim
591 index, columns = self.new_axes
592 df = cons(data, index=index, copy=self.copy)
593 df.columns = columns
594 return df.__finalize__(self, method="concat")
596 # combine block managers
597 else:
598 sample = cast("DataFrame", self.objs[0])
600 mgrs_indexers = []
601 for obj in self.objs:
602 indexers = {}
603 for ax, new_labels in enumerate(self.new_axes):
604 # ::-1 to convert BlockManager ax to DataFrame ax
605 if ax == self.bm_axis:
606 # Suppress reindexing on concat axis
607 continue
609 # 1-ax to convert BlockManager axis to DataFrame axis
610 obj_labels = obj.axes[1 - ax]
611 if not new_labels.equals(obj_labels):
612 indexers[ax] = obj_labels.get_indexer(new_labels)
614 mgrs_indexers.append((obj._mgr, indexers))
616 new_data = concatenate_managers(
617 mgrs_indexers, self.new_axes, concat_axis=self.bm_axis, copy=self.copy
618 )
619 if not self.copy:
620 new_data._consolidate_inplace()
622 cons = sample._constructor
623 return cons(new_data).__finalize__(self, method="concat")
625 def _get_result_dim(self) -> int:
626 if self._is_series and self.bm_axis == 1:
627 return 2
628 else:
629 return self.objs[0].ndim
631 def _get_new_axes(self) -> list[Index]:
632 ndim = self._get_result_dim()
633 return [
634 self._get_concat_axis if i == self.bm_axis else self._get_comb_axis(i)
635 for i in range(ndim)
636 ]
638 def _get_comb_axis(self, i: int) -> Index:
639 data_axis = self.objs[0]._get_block_manager_axis(i)
640 return get_objs_combined_axis(
641 self.objs,
642 axis=data_axis,
643 intersect=self.intersect,
644 sort=self.sort,
645 copy=self.copy,
646 )
648 @cache_readonly
649 def _get_concat_axis(self) -> Index:
650 """
651 Return index to be used along concatenation axis.
652 """
653 if self._is_series:
654 if self.bm_axis == 0:
655 indexes = [x.index for x in self.objs]
656 elif self.ignore_index:
657 idx = default_index(len(self.objs))
658 return idx
659 elif self.keys is None:
660 names: list[Hashable] = [None] * len(self.objs)
661 num = 0
662 has_names = False
663 for i, x in enumerate(self.objs):
664 if not isinstance(x, ABCSeries):
665 raise TypeError(
666 f"Cannot concatenate type 'Series' with "
667 f"object of type '{type(x).__name__}'"
668 )
669 if x.name is not None:
670 names[i] = x.name
671 has_names = True
672 else:
673 names[i] = num
674 num += 1
675 if has_names:
676 return Index(names)
677 else:
678 return default_index(len(self.objs))
679 else:
680 return ensure_index(self.keys).set_names(self.names)
681 else:
682 indexes = [x.axes[self.axis] for x in self.objs]
684 if self.ignore_index:
685 idx = default_index(sum(len(i) for i in indexes))
686 return idx
688 if self.keys is None:
689 if self.levels is not None:
690 raise ValueError("levels supported only when keys is not None")
691 concat_axis = _concat_indexes(indexes)
692 else:
693 concat_axis = _make_concat_multiindex(
694 indexes, self.keys, self.levels, self.names
695 )
697 self._maybe_check_integrity(concat_axis)
699 return concat_axis
701 def _maybe_check_integrity(self, concat_index: Index):
702 if self.verify_integrity:
703 if not concat_index.is_unique:
704 overlap = concat_index[concat_index.duplicated()].unique()
705 raise ValueError(f"Indexes have overlapping values: {overlap}")
708def _concat_indexes(indexes) -> Index:
709 return indexes[0].append(indexes[1:])
712def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiIndex:
714 if (levels is None and isinstance(keys[0], tuple)) or (
715 levels is not None and len(levels) > 1
716 ):
717 zipped = list(zip(*keys))
718 if names is None:
719 names = [None] * len(zipped)
721 if levels is None:
722 _, levels = factorize_from_iterables(zipped)
723 else:
724 levels = [ensure_index(x) for x in levels]
725 else:
726 zipped = [keys]
727 if names is None:
728 names = [None]
730 if levels is None:
731 levels = [ensure_index(keys).unique()]
732 else:
733 levels = [ensure_index(x) for x in levels]
735 for level in levels:
736 if not level.is_unique:
737 raise ValueError(f"Level values not unique: {level.tolist()}")
739 if not all_indexes_same(indexes) or not all(level.is_unique for level in levels):
740 codes_list = []
742 # things are potentially different sizes, so compute the exact codes
743 # for each level and pass those to MultiIndex.from_arrays
745 for hlevel, level in zip(zipped, levels):
746 to_concat = []
747 for key, index in zip(hlevel, indexes):
748 # Find matching codes, include matching nan values as equal.
749 mask = (isna(level) & isna(key)) | (level == key)
750 if not mask.any():
751 raise ValueError(f"Key {key} not in level {level}")
752 i = np.nonzero(mask)[0][0]
754 to_concat.append(np.repeat(i, len(index)))
755 codes_list.append(np.concatenate(to_concat))
757 concat_index = _concat_indexes(indexes)
759 # these go at the end
760 if isinstance(concat_index, MultiIndex):
761 levels.extend(concat_index.levels)
762 codes_list.extend(concat_index.codes)
763 else:
764 codes, categories = factorize_from_iterable(concat_index)
765 levels.append(categories)
766 codes_list.append(codes)
768 if len(names) == len(levels):
769 names = list(names)
770 else:
771 # make sure that all of the passed indices have the same nlevels
772 if not len({idx.nlevels for idx in indexes}) == 1:
773 raise AssertionError(
774 "Cannot concat indices that do not have the same number of levels"
775 )
777 # also copies
778 names = list(names) + list(get_unanimous_names(*indexes))
780 return MultiIndex(
781 levels=levels, codes=codes_list, names=names, verify_integrity=False
782 )
784 new_index = indexes[0]
785 n = len(new_index)
786 kpieces = len(indexes)
788 # also copies
789 new_names = list(names)
790 new_levels = list(levels)
792 # construct codes
793 new_codes = []
795 # do something a bit more speedy
797 for hlevel, level in zip(zipped, levels):
798 hlevel = ensure_index(hlevel)
799 mapped = level.get_indexer(hlevel)
801 mask = mapped == -1
802 if mask.any():
803 raise ValueError(f"Values not found in passed level: {hlevel[mask]!s}")
805 new_codes.append(np.repeat(mapped, n))
807 if isinstance(new_index, MultiIndex):
808 new_levels.extend(new_index.levels)
809 new_codes.extend([np.tile(lab, kpieces) for lab in new_index.codes])
810 else:
811 new_levels.append(new_index.unique())
812 single_codes = new_index.unique().get_indexer(new_index)
813 new_codes.append(np.tile(single_codes, kpieces))
815 if len(new_names) < len(new_levels):
816 new_names.extend(new_index.names)
818 return MultiIndex(
819 levels=new_levels, codes=new_codes, names=new_names, verify_integrity=False
820 )