U
    cd                     @  s6  d dl mZ d dlZd dlZd dlmZmZmZ d dlZ	d dl
mZmZ d dlmZ d dlmZmZmZmZ d dlmZ d dlmZmZ d d	lmZmZmZmZ d d
lm Z m!Z! d dl"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*  m+Z, d dl-m.Z.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; erXd dl<m=Z= d dl7m>Z> dddddddZ?ddddd Z@dddddd!d"ZAdd#d#d$d%d&ZBd'd(d)d*d+ZCG d,d- d-ZDd.dddd/d0d1ZEd2dd3d4d5ZFd6d2d7d8d9ZGd.dd7d:d;ZHdd<d=d>ZId-dd-d?d@dAZJddBdCdDZKdS )E    )annotationsN)TYPE_CHECKINGSequencecast)NaT	internals)NA)	ArrayLikeDtypeObjManagerShape)cache_readonly)ensure_dtype_can_hold_nafind_common_type)is_1d_only_ea_dtypeis_dtype_equal	is_scalarneeds_i8_conversion)cast_to_common_typeconcat_compat)DatetimeTZDtypeExtensionDtype)is_valid_na_for_dtypeisnaisna_all)DatetimeArrayExtensionArray)SparseDtype)ensure_wrapped_if_datetimelike)ArrayManagerNullArrayProxy)ensure_block_shapenew_block_2d)BlockManager)Index)Blockzlist[Index]intboolr   )axesconcat_axiscopyreturnc              	     s   g  | D ]l\}}d}|  D ]4\}}|j|| ||ddd}|dkr|dk	rd}q|rj|dkrj|sj| } | q|dkr fddtt d jD }	n&|dksttt	j
d	d  D }	t|	|d |d gdd
}
|
S )z
    Concatenate array managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (ArrayManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    ArrayManager
    FT)axis
allow_dupsuse_na_proxy   Nr   c                   s,   g | ]$ t  fd dttD qS )c                   s   g | ]}| j   qS  arrays).0i)jmgrsr0   @/tmp/pip-unpacked-wheel-g7fro6k3/pandas/core/internals/concat.py
<listcomp>l   s     z:_concatenate_array_managers.<locals>.<listcomp>.<listcomp>)concat_arraysrangelen)r3   r6   )r5   r7   r8   k   s   z/_concatenate_array_managers.<locals>.<listcomp>c                 S  s   g | ]
}|j qS r0   r1   )r3   mgrr0   r0   r7   r8   r   s     )Zverify_integrity)itemsreindex_indexerr*   appendr:   r;   r2   AssertionErrorlist	itertoolschainfrom_iterabler   )mgrs_indexersr(   r)   r*   r=   indexersZaxis1_made_copyaxindexerr2   new_mgrr0   r<   r7   _concatenate_array_managersH   s0        
rK   rB   r	   )	to_concatr+   c                   s   dd | D }dd |D }t |dk}|r8|d j n6tdd |D r\tt|g  ntd	d |D   fd
d| D } t| d trt| d }|	| S t
| }t |dkrdd |D }t |dkrd|kr|t}|S )a  
    Alternative for concat_compat but specialized for use in the ArrayManager.

    Differences: only deals with 1D arrays (no axis keyword), assumes
    ensure_wrapped_if_datetimelike and does not skip empty arrays to determine
    the dtype.
    In addition ensures that all NullArrayProxies get replaced with actual
    arrays.

    Parameters
    ----------
    to_concat : list of arrays

    Returns
    -------
    np.ndarray or ExtensionArray
    c                 S  s   g | ]}t |ts|qS r0   )
isinstancer    r3   xr0   r0   r7   r8      s     
 z!concat_arrays.<locals>.<listcomp>c                 S  s   h | ]
}|j qS r0   dtyperN   r0   r0   r7   	<setcomp>   s     z concat_arrays.<locals>.<setcomp>r/   r   c                 s  s$   | ]}|j d kot|tjV  qdS ))r4   ubN)kindrM   nprQ   rN   r0   r0   r7   	<genexpr>   s     z concat_arrays.<locals>.<genexpr>c                 S  s   g | ]
}|j qS r0   rP   r3   Zarrr0   r0   r7   r8      s     c                   s*   g | ]"}t |tr| nt| qS r0   )rM   r    Zto_arrayr   rX   Ztarget_dtyper0   r7   r8      s   c                 S  s   h | ]}|j jqS r0   )rQ   rU   )r3   objr0   r0   r7   rR      s     rT   )r;   rQ   allrV   r   rB   rM   r   typeZ_concat_same_typeconcatenateastypeobject)rL   Zto_concat_no_proxydtypesZsingle_dtypeclsresultkindsr0   rY   r7   r9   x   s*    



r9   c                 C  sB  t | d d tr t| |||S t|| } dd | D }t||}g }|D ]\}}|d }	|	j}
t|dkr|d js|
j}|r|	 }n|
 }d}nnt|rdd |D }|
jstj|dd}nt|dd}t|dd	}t|}|
jj|jk}nt|||d
}d}|r|
j||d}nt||d}|| qJtt||S )z
    Concatenate block managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    BlockManager
    r   c                 S  s   g | ]\}}t ||qS r0   )_get_mgr_concatenation_plan)r3   r=   rG   r0   r0   r7   r8      s    z(concatenate_managers.<locals>.<listcomp>r/   Tc                 S  s   g | ]}|j jqS r0   )blockvaluesr3   Zjur0   r0   r7   r8      s     r,      )ndim)r*   F)	placement)rM   r   rK   _maybe_reindex_columns_na_proxy_combine_concat_plansre   r;   rG   rf   r*   view_is_uniform_join_unitsis_extensionrV   r]   r   r!   r   rQ   _concatenate_join_unitsZmake_block_same_classr"   r@   r#   tuple)rF   r(   r)   r*   Zconcat_plansZconcat_planblocksrk   
join_unitsunitblkrf   ZfastpathvalsrT   r0   r0   r7   concatenate_managers   s@    


rx   z0list[tuple[BlockManager, dict[int, np.ndarray]]])r(   rF   r+   c              
   C  sj   g }|D ]\\}}d|krV|j | d |d dddddd}| }|d= |||f q|||f q|S )z
    Reindex along columns so that all of the BlockManagers being concatenated
    have matching columns.

    Columns added in this reindexing have dtype=np.void, indicating they
    should be ignored when choosing a column's final dtype.
    r   FT)r,   r*   Z
only_slicer-   r.   )r?   r*   r@   )r(   rF   Znew_mgrs_indexersr=   rG   rJ   Znew_indexersr0   r0   r7   rl      s"    
	rl   r#   zdict[int, np.ndarray])r=   rG   c                 C  sN  t | j}| D ]\}}t|||< qt|}d|ks<t| jr`| jd }|jt	|||fgS | j
}| j}g }	tj|ddD ]\}
}|jst|
dkst| }t |}t||d< t|}| j|
 }||j }t|t|jko|jjo|jjjdkpt|dk }|r&|dd n||d< t	|||}|	||f q~|	S )z
    Construct concatenation plan for given block manager and indexers.

    Parameters
    ----------
    mgr : BlockManager
    indexers : dict of {axis: indexer}

    Returns
    -------
    plan : list of (BlockPlacement, JoinUnit) tuples

    r   F)groupr/   N)rB   shaper>   r;   rr   rA   Zis_single_blockrs   Zmgr_locsJoinUnitblknosblklocslibinternalsZget_blkno_placementsZis_slice_liker*   rI   as_slicesteprV   Zdiffr[   popr@   )r=   rG   Zmgr_shape_listrH   rI   Z	mgr_shaperv   r}   r~   planZblkno
placementsZjoin_unit_indexersZ
shape_listr{   Zax0_blk_indexerZunit_no_ax0_reindexingru   r0   r0   r7   rd     s<    




rd   c                   @  s   e Zd ZddddddZddd	d
ZeddddZeddddZdddddZeddddZ	dddddZ
dS )r|   Nr%   r   )re   r{   c                 C  s"   |d kri }|| _ || _|| _d S Nre   rG   r{   )selfre   r{   rG   r0   r0   r7   __init__b  s
    zJoinUnit.__init__strr+   c                 C  s$   t | j dt| j d| j dS )N(z, ))r\   __name__reprre   rG   )r   r0   r0   r7   __repr__k  s    zJoinUnit.__repr__r'   c                 C  s&   | j  D ]}|dk r
 dS q
dS )Nrz   TF)rG   rf   any)r   rI   r0   r0   r7   needs_fillingn  s    zJoinUnit.needs_fillingr
   c                 C  s2   | j }|jjjdkrtd| js(|jS t|jS )NVzBlock is None, no dtype)re   rf   rQ   rU   rA   r   r   )r   rv   r0   r0   r7   rQ   w  s    zJoinUnit.dtype)rQ   r+   c                   s   | j s
dS | jjjdkrdS | jtkrL| jj}t fdd|jddD S | jj}|t	krlt
| j sldS |tkrt rdS t| S )z
        Check that we are all-NA of a type/dtype that is compatible with this dtype.
        Augments `self.is_na` with an additional check of the type of NA values.
        Fr   Tc                 3  s   | ]}t | V  qd S r   )r   rN   rP   r0   r7   rW     s     z,JoinUnit._is_valid_na_for.<locals>.<genexpr>Korder)is_nare   rQ   rU   r_   rf   r[   ravel
fill_valuer   r   r   r   r   )r   rQ   rf   na_valuer0   rP   r7   _is_valid_na_for  s    
zJoinUnit._is_valid_na_forc                 C  s   | j }|jjdkrdS |js dS |j}|jdkr4dS t|jtrDdS |jdkrr|d }t	|rft
|sjdS t|S |d d }t	|rt
|sdS tdd |D S d S )Nr   TFr   r/   c                 s  s   | ]}t |V  qd S r   )r   )r3   rowr0   r0   r7   rW     s     z!JoinUnit.is_na.<locals>.<genexpr>)re   rQ   rU   Z_can_hold_narf   sizerM   r   rj   r   r   r   r[   )r   rv   rf   valr0   r0   r7   r     s&    

zJoinUnit.is_nar	   )empty_dtyper+   c                 C  s  |d kr,| j jjdkr,| j j}| j  }n|}| |rp| j j}|tdkrz| j jjdd}t	|rz|d d krzd }t
|trt| j|j}t||dS t|rt||r| jrn`tt|}| }|jg |d}| j\}	}
|	dkst|	dtj|
ftjd }|j|d	|d
S nRt
|trR| }|j| j|d}||d d < |S tj| j|d}|| |S | js| j js| j jS | j jr| j  tdj}n| j j}| js|! }n&| j" D ]\}}t#j$|||d}q|S )Nr   r_   r   r   r   rP   r/   rz   T)Z
allow_fillr   )r{   rQ   rh   )%re   rQ   rU   r   Z
get_valuesr   rV   rf   r   r;   rM   r   fullr{   valuer   r   r   rG   r   r   Zconstruct_array_typeZ_from_sequencerA   ZonesZintpZtake_emptyemptyfillZ_can_consolidateZis_boolr^   rn   r>   algosZtake_nd)r   r   upcasted_nar   rf   Z	blk_dtypeZi8valuesra   Zmissing_arrZncolsZnrowsZ	empty_arrrH   rI   r0   r0   r7   get_reindexed_values  sZ    



  


zJoinUnit.get_reindexed_values)N)r   
__module____qualname__r   r   r   r   rQ   r   r   r   r0   r0   r0   r7   r|   a  s   		r|   zlist[JoinUnit])rt   r)   r*   r+   c                   s   |dkrt | dkrtdt|  tdd | D }t | fdd| D }t |dkr|d }|rt|tjr|jdk	r|	 }q|	 }nFtd	d |D rd
d |D }t
|ddd}t|d}nt
||d}|S )zI
    Concatenate values from several join units along selected axis.
    r   r/   z$Concatenating join units along axis0c                 s  s   | ]}|j jjd kV  qdS r   Nre   rQ   rU   r3   ru   r0   r0   r7   rW     s     z*_concatenate_join_units.<locals>.<genexpr>c                   s   g | ]}|j  d qS )r   r   )r   rg   r   r0   r7   r8     s   z+_concatenate_join_units.<locals>.<listcomp>Nc                 s  s   | ]}t |jV  qd S r   r   rQ   r3   tr0   r0   r7   rW   /  s     c                 S  s*   g | ]"}t |jr|n|d ddf qS )r   Nr   r   r0   r0   r7   r8   7  s   T)r,   Zea_compat_axisri   rh   )r;   rA   _get_empty_dtyper   _dtype_to_na_valuerM   rV   Zndarraybaser*   r   r!   )rt   r)   r*   has_none_blocksrL   Zconcat_valuesr0   r   r7   rq     s.    



rq   r
   rQ   r   c                 C  sv   t | tr| jS | jdkr$| dS | jdkr8| dS | jdkrFdS | jdkr^|sXdS tjS | jdkrntjS tdS )	z2
    Find the NA value to go with this dtype.
    )mMr   )fcNaNrT   N)r4   rS   O)rM   r   r   rU   r\   rV   nanNotImplementedErrorr   r0   r0   r7   r   F  s    







r   zSequence[JoinUnit])rt   r+   c                 C  s   t | dkr| d j}|jS t| r4| d jj}|S tdd | D }dd | D }t |sjdd | D }t|}|r~t|}|S )z
    Return dtype and N/A values to use when concatenating specified units.

    Returned N/A value may be None which means there was no casting involved.

    Returns
    -------
    dtype
    r/   r   c                 s  s   | ]}|j jjd kV  qdS r   r   r   r0   r0   r7   rW   o  s     z#_get_empty_dtype.<locals>.<genexpr>c                 S  s   g | ]}|j s|jqS r0   )r   rQ   r   r0   r0   r7   r8   q  s      z$_get_empty_dtype.<locals>.<listcomp>c                 S  s    g | ]}|j jjd kr|jqS )r   r   r   r0   r0   r7   r8   s  s      )r;   re   rQ   _is_uniform_reindexr   r   r   )rt   rv   r   r   r`   rQ   r0   r0   r7   r   ]  s    

r   c                   sv   | d j   jjdkrdS t fdd| D ott fdd| D ottdd | D ottdd | D ott| d	kS )
z
    Check if the join units consist of blocks of uniform type that can
    be concatenated using Block.concat_same_type instead of the generic
    _concatenate_join_units (which uses `concat_compat`).

    r   r   Fc                 3  s    | ]}t |jt  kV  qd S r   )r\   re   rg   firstr0   r7   rW     s     z)_is_uniform_join_units.<locals>.<genexpr>c                 3  s,   | ]$}t |jj jp"|jjjd kV  qdS ))rT   r4   rS   N)r   re   rQ   rU   rg   r   r0   r7   rW     s   c                 s  s   | ]}|j  p|jjV  qd S r   )r   re   rp   rg   r0   r0   r7   rW     s     c                 s  s   | ]}|j  V  qd S r   )rG   rg   r0   r0   r7   rW     s     r/   )re   rQ   rU   r[   r;   rt   r0   r   r7   ro   {  s    

ro   r   c                 C  s(   t dd | D o&tdd | D dkS )Nc                 s  s   | ]}|j jV  qd S r   )re   rp   rg   r0   r0   r7   rW     s     z&_is_uniform_reindex.<locals>.<genexpr>c                 S  s   h | ]}|j jjqS r0   )re   rQ   namerg   r0   r0   r7   rR     s     z&_is_uniform_reindex.<locals>.<setcomp>r/   )r[   r;   r   r0   r0   r7   r     s    r   )	join_unitlengthr+   c                 C  s   d| j krF| j }| jdkr d}q| jt|d}| jt|| _n>| j}t| j }|d |d |d< | j d d| | j d< | jd | f| jdd  }|f| jdd  | _t|||dS )z
    Reduce join_unit's shape along item axis to length.

    Extra items that didn't fit are returned as a separate block.
    r   Nr/   r   )rG   re   Zgetitem_blockslicer*   r{   r|   )r   r   Zextra_indexersZextra_blockZextra_shaper0   r0   r7   _trim_join_unit  s    

r   )r)   c                 #  s  t | dkr2| d D ]}|d |d gfV  qn~|dkrd}| D ]@}d}|D ]\}}|||gfV  |}qN|dk	rB||jj7 }qBn(dg  fdd}ttt| } tt|| }	 d t |	kr d dkrtdt|	 \}
}ttt |
}t	|t
| }}||kr4|
d |fV  t|| |	dd< qd}dgt |	 }t|	D ]T\}\}}|||< t ||kr||d t||f|	|< n|}|| | |	|< qN||fV  qdS )z`
    Combine multiple concatenation plans into one.

    existing_plan is updated in-place.
    r/   r   Nc                   s&   t | d }|d kr" d  d7  < |S )Nr   r/   )next)seqretvalZ	num_endedr0   r7   _next_or_none  s    
z,_combine_concat_plans.<locals>._next_or_nonezPlan shapes are not aligned)r;   addr   stoprB   mapiter
ValueErrorzipminmax	enumerater   )Zplansr)   poffsetr   Zlast_plcZplcru   r   Z
next_itemsr   ZunitslengthsZmin_lenmax_lenZyielded_placementZyielded_unitsr4   r0   r   r7   rm     sD    
rm   )L
__future__r   r*   rC   typingr   r   r   ZnumpyrV   Zpandas._libsr   r   r   Zpandas._libs.missingr   Zpandas._typingr	   r
   r   r   Zpandas.util._decoratorsr   Zpandas.core.dtypes.castr   r   Zpandas.core.dtypes.commonr   r   r   r   Zpandas.core.dtypes.concatr   r   Zpandas.core.dtypes.dtypesr   r   Zpandas.core.dtypes.missingr   r   r   Zpandas.core.algorithmscoreZ
algorithmsr   Zpandas.core.arraysr   r   Zpandas.core.arrays.sparser   Zpandas.core.constructionr   Z#pandas.core.internals.array_managerr   r    Zpandas.core.internals.blocksr!   r"   Zpandas.core.internals.managersr#   Zpandasr$   r%   rK   r9   rx   rl   rd   r|   rq   r   r   ro   r   r   rm   r0   r0   r0   r7   <module>   sJ   08F K /7#