U
    cia                     @  s  d Z ddlmZ ddlZddlmZmZ ddlmZ ddl	Z	ddl
mZmZ ddlZddlmZ ddlmZ ddlm  mZ dd	lmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& dd
l'm(Z(m)Z) ddl*m+Z+ ddl,m-  m.  m/Z/ dddddZ0dddddZ1dddddZ2dd Z3dd Z4e4e2e1e0e(fdddddZ5dd Z6e6e7Z8e6e9Z:e;dd d d e<eD D Z=e=fd!d"Z>e;e?d#d$ e=Z@e>ejAZBe>ejCZDe>ejEZFe>ejGZHe>ejIZJe>ejKZLe>ejMZNe>ejOZPe>ejQZRe>ejSZTe>ejUZVe>ejZWe>ejXZYes<e>ejZZ[e;d%d&d'gZ\e;d(d)d*d+d,d-d.d/d0d1d2gZ]eDeBB eTB eVB eWB eYB eHB e]B e\ Z^e@e^ e\B Z_e^e_@ Z`d3e` Zae`rtbeadd4d5d6d7Zced8Zdd9d:d;d<d=Zed>d? Zfee%d@ZgdAdB Zheee^ehegG dCdD dDejiZje;dEdFdGdHgZke;ee Zleee^ekB eJe;dFdIdGdHdJgB  G dKdL dLejZmeee^ekB e;dMgB G dNdO dOejZnG dPd' d'ZoenemdQZpdS )Rz
:func:`~pandas.eval` parsers.
    )annotationsN)partialreduce	iskeyword)CallableTypeVar)PY39)UndefinedVariableError)ARITH_OPS_SYMSBOOL_OPS_SYMSCMP_OPS_SYMS	LOCAL_TAGMATHOPS
REDUCTIONSUNARY_OPS_SYMSBinOpConstantDivFuncNodeOpTermUnaryOpis_term)clean_backtick_quoted_tokstokenize_string)Scopeztuple[int, str])tokreturnc                 C  s   | \}}||dkrdn|fS )a`  
    Rewrite the assignment operator for PyTables expressions that use ``=``
    as a substitute for ``==``.

    Parameters
    ----------
    tok : tuple of int, str
        ints correspond to the all caps constants in the tokenize module

    Returns
    -------
    tuple of int, str
        Either the input or token or the replacement values
    === r   toknumtokvalr!   r!   @/tmp/pip-unpacked-wheel-g7fro6k3/pandas/core/computation/expr.py_rewrite_assign2   s    r&   c                 C  sF   | \}}|t jkr>|dkr$t jdfS |dkr6t jdfS ||fS ||fS )aj  
    Replace ``&`` with ``and`` and ``|`` with ``or`` so that bitwise
    precedence is changed to boolean precedence.

    Parameters
    ----------
    tok : tuple of int, str
        ints correspond to the all caps constants in the tokenize module

    Returns
    -------
    tuple of int, str
        Either the input or token or the replacement values
    &and|or)tokenizeOPNAMEr"   r!   r!   r%   _replace_booleansE   s    


r.   c                 C  s,   | \}}|t jkr$|dkr$t jtfS ||fS )a(  
    Replace local variables with a syntactically valid name.

    Parameters
    ----------
    tok : tuple of int, str
        ints correspond to the all caps constants in the tokenize module

    Returns
    -------
    tuple of int, str
        Either the input or token or the replacement values

    Notes
    -----
    This is somewhat of a hack in that we rewrite a string such as ``'@a'`` as
    ``'__pd_eval_local_a'`` by telling the tokenizer that ``__pd_eval_local_``
    is a ``tokenize.OP`` and to replace the ``'@'`` symbol with it.
    @)r+   r,   r   r"   r!   r!   r%   _replace_locals^   s    
r0   c                   s    fddS )z
    Compose 2 callables.
    c                    s    | |S Nr!   )argskwargsfgr!   r%   <lambda>|       z_compose2.<locals>.<lambda>r!   r4   r!   r4   r%   	_compose2x   s    r9   c                  G  s   t | dkstdtt| S )z&
    Compose 2 or more callables.
       z.At least 2 callables must be passed to compose)lenAssertionErrorr   r9   )funcsr!   r!   r%   _compose   s    r>   str)sourcer   c                   s,   t  stdt fddt| D S )a  
    Compose a collection of tokenization functions.

    Parameters
    ----------
    source : str
        A Python source code string
    f : callable
        This takes a tuple of (toknum, tokval) as its argument and returns a
        tuple with the same structure but possibly different elements. Defaults
        to the composition of ``_rewrite_assign``, ``_replace_booleans``, and
        ``_replace_locals``.

    Returns
    -------
    str
        Valid Python source code

    Notes
    -----
    The `f` parameter can be any callable that takes *and* returns input of the
    form ``(toknum, tokval)``, where ``toknum`` is one of the constants from
    the ``tokenize`` module and ``tokval`` is a string.
    zf must be callablec                 3  s   | ]} |V  qd S r1   r!   .0xr5   r!   r%   	<genexpr>   s     z_preparse.<locals>.<genexpr>)callabler<   r+   
untokenizer   r@   r5   r!   rD   r%   	_preparse   s    rI   c                   s    fddS )zO
    Factory for a type checking function of type ``t`` or tuple of types.
    c                   s   t | j S r1   )
isinstancevaluerC   tr!   r%   r7      r8   z_is_type.<locals>.<lambda>r!   rM   r!   rM   r%   _is_type   s    rO   c                 c  s(   | ] }t |trt|tjr|V  qd S r1   )rJ   type
issubclassastASTrB   noder!   r!   r%   rE      s   
 rE   c                 c  s   | ]}t t|V  qd S r1   )getattrrR   )rB   namer!   r!   r%   rE      s     c                   s    fdd|D }t |S )zE
    Filter out AST nodes that are subclasses of ``superclass``.
    c                 3  s   | ]}t | r|jV  qd S r1   )rQ   __name__rT   
superclassr!   r%   rE      s     
 z _filter_nodes.<locals>.<genexpr>)	frozenset)rZ   Z	all_nodesZ
node_namesr!   rY   r%   _filter_nodes   s    r\   c                 C  s   | j S r1   )rX   rL   r!   r!   r%   r7      r8   r7   ZAssignModuleExprZYieldZGeneratorExpZIfExpZDictCompZSetCompReprLambdaSetrS   ZIsZIsNotz$cannot both support and not support zCallable[(Ellipsis, None)])	node_namer   c                   s    fdd}|S )zV
    Return a function that raises a NotImplementedError with a passed node name.
    c                   s   t d  dd S )N'z' nodes are not implemented)NotImplementedError)selfr2   r3   rb   r!   r%   r5     s    z _node_not_implemented.<locals>.fr!   )rb   r5   r!   rf   r%   _node_not_implemented   s    rg   _Tzset[str]zCallable[[type[_T]], type[_T]])nodesr   c                   s   ddd fdd}|S )z
    Decorator to disallow certain nodes from parsing. Raises a
    NotImplementedError instead.

    Returns
    -------
    callable
    ztype[_T])clsr   c                   sB   d| _  D ]2}t|}d| }|  j |f7  _ t| || q
| S )Nr!   visit_)unsupported_nodesrg   setattr)rj   rU   Z
new_methodrW   ri   r!   r%   
disallowed  s    
zdisallow.<locals>.disallowedr!   )ri   ro   r!   rn   r%   disallow  s    
rp   c                   s    fdd}|S )zw
    Return a function to create an op class with its symbol already passed.

    Returns
    -------
    callable
    c                   s   t  f||S )z
        Return a partial function with an Op subclass with an operator already passed.

        Returns
        -------
        callable
        )r   )re   rU   r2   r3   op_class	op_symbolr!   r%   r5   2  s    z_op_maker.<locals>.fr!   )rr   rs   r5   r!   rq   r%   	_op_maker)  s    	
rt   )binaryZunaryc                   s    fdd}|S )z9
    Decorator to add default implementation of ops.
    c                   sl      D ]^\}}t| | d}t| | d}|D ]0}|| }|d k	r4t||}t| d| | q4q| S )NZ_opsZ_op_nodes_maprk   )itemsrV   rt   rm   )rj   Zop_attr_namerr   opsZops_mapopZop_nodeZmade_op
op_classesr!   r%   r5   G  s    
zadd_ops.<locals>.fr!   )rz   r5   r!   ry   r%   add_opsB  s    r{   c                   @  s  e Zd ZU dZeZded< eZe	e
 e ZdZeeeeZeZdZdd eeeD ZejejejejejejejejiZded	< efd
dddZdd Zdd Zdd Zdd Z dHddZ!dd Z"dd Z#dIdd Z$d!d" Z%d#d$ Z&d%d& Z'd'd( Z(d)dd*d+Z)d)dd,d-Z*d)dd.d/Z+d0d1 Z,d2d3 Z-e-Z.d4d5 Z/d6d7 Z0d8d9 Z1d:d; Z2d<d= Z3dJd>d?Z4d@dA Z5dBdC Z6dDdE Z7dFdG Z8dS )KBaseExprVisitorz
    Custom ast walker. Parsers of other engines should subclass this class
    if necessary.

    Parameters
    ----------
    env : Scope
    engine : str
    parser : str
    preparser : callable
    z
type[Term]
const_type)ZGtZLtZGtEZLtEEqNotEqInNotInZBitAndZBitOrAndOrZAddZSubZMultNZPowZFloorDivZMod)ZUAddZUSubZInvertNotc                 C  s   i | ]\}}||qS r!   r!   )rB   kvr!   r!   r%   
<dictcomp>  s      zBaseExprVisitor.<dictcomp>ztuple[str, ...]rl   Noner   c                 C  s"   || _ || _|| _|| _d | _d S r1   )envengineparser	preparserassignerre   r   r   r   r   r!   r!   r%   __init__  s
    zBaseExprVisitor.__init__c              
   K  s   t |trl| |}ztt|}W nB tk
rj } z$tdd | D rVd|_	|W 5 d }~X Y nX dt
|j }t| |}||f|S )Nc                 s  s   | ]}t |V  qd S r1   r   rA   r!   r!   r%   rE     s     z(BaseExprVisitor.visit.<locals>.<genexpr>z4Python keyword not valid identifier in numexpr queryrk   )rJ   r?   r   rR   Zfix_missing_locationsparseSyntaxErroranysplitmsgrP   rX   rV   )re   rU   r3   cleanemethodvisitorr!   r!   r%   visit  s    


zBaseExprVisitor.visitc                 K  s.   t |jdkrtd|jd }| j|f|S )Nr:   z#only a single expression is allowedr   )r;   bodyr   r   )re   rU   r3   exprr!   r!   r%   visit_Module  s    
zBaseExprVisitor.visit_Modulec                 K  s   | j |jf|S r1   r   rK   re   rU   r3   r!   r!   r%   
visit_Expr  s    zBaseExprVisitor.visit_Exprc                 C  s   |j }t|}t|rt|r|| jkrtt||f\}}tt||f\}}	|s\|s\|s\|	rh| j|  }|	r| j|j	g}
| 
|
| j}|r| j|j	g}
| 
|
| j}| |}||||fS r1   )rx   rP   r   rewrite_mapmap_is_list_is_strr   add_tmprK   	term_typer   )re   rU   leftrightZop_instanceZop_typeZ	left_listZ
right_listZleft_strZ	right_strrW   rx   r!   r!   r%   _rewrite_membership_op  s    
z&BaseExprVisitor._rewrite_membership_opNc                 C  sR   |d kr| j |jdd}|d kr0| j |jdd}| |||\}}}}||||fS )Nr   )sider   )r   r   r   r   )re   rU   r   r   rx   rr   r!   r!   r%   _maybe_transform_eq_ne  s    z&BaseExprVisitor._maybe_transform_eq_nec                 C  s   t t j}|jrNt|drN|jsN|j|krN| jt |j}| 	|| j}|jrt|dr|js|j|kr| jt |j}| 	|| j}||fS )NrK   )
npZdtypeZfloat32Z	is_scalarhasattrreturn_typer   r   rK   r   )re   r   r   Zf32rW   r!   r!   r%   _maybe_downcast_constants  s,    z)BaseExprVisitor._maybe_downcast_constantsc                 C  s   | | j| j| j| j|S r1   )evaluater   r   r   r   )re   binopeval_in_pythonr!   r!   r%   _maybe_eval  s        zBaseExprVisitor._maybe_evalinznot inr    z!=<>z<=z>=c                 C  s   |||}|j r2td|j d|j d|j d| jdkrl|jtkrRt|dds^t|ddrl| || jS |j|kr| ||S | jdkrt|dd t	kst|dd t	kr| ||| S |S )	Nz unsupported operand type(s) for z: 'z' and 'rc   ZpytablesZis_datetimeFr   )
Zhas_invalid_return_type	TypeErrorrx   rP   r   r   rV   r   
binary_opsobject)re   rx   rr   lhsrhsr   Zmaybe_eval_in_pythonresr!   r!   r%   _maybe_evaluate_binop  s,    	





z%BaseExprVisitor._maybe_evaluate_binopc                 K  s2   |  |\}}}}| ||\}}| ||||S r1   )r   r   r   )re   rU   r3   rx   rr   r   r   r!   r!   r%   visit_BinOp  s    zBaseExprVisitor.visit_BinOpc                 K  s   dd S )Nc                 S  s
   t | |S r1   )r   )r   r   r!   r!   r%   r7     r8   z+BaseExprVisitor.visit_Div.<locals>.<lambda>r!   r   r!   r!   r%   	visit_Div  s    zBaseExprVisitor.visit_Divc                 K  s    |  |j}|  |j}||S r1   )r   rx   operand)re   rU   r3   rx   r   r!   r!   r%   visit_UnaryOp  s    zBaseExprVisitor.visit_UnaryOpc                 K  s   | j |j| jf|S r1   )r   idr   r   r!   r!   r%   
visit_Name$  s    zBaseExprVisitor.visit_Namer   c                 K  s   |  |j| jS r1   )r}   rK   r   r   r!   r!   r%   visit_NameConstant'  s    z"BaseExprVisitor.visit_NameConstantc                 K  s   |  |j| jS r1   r}   nr   r   r!   r!   r%   	visit_Num*  s    zBaseExprVisitor.visit_Numc                 K  s   |  |j| jS r1   r   r   r!   r!   r%   visit_Constant-  s    zBaseExprVisitor.visit_Constantc                 K  s   | j |j}| || j S r1   )r   r   sr   re   rU   r3   rW   r!   r!   r%   	visit_Str0  s    zBaseExprVisitor.visit_Strc                   s*    j  fdd|jD } | j S )Nc                   s   g | ]}  | jqS r!   )r   r   )rB   r   re   r!   r%   
<listcomp>5  s     z.BaseExprVisitor.visit_List.<locals>.<listcomp>)r   r   Zeltsr   r   r!   r   r%   
visit_List4  s    zBaseExprVisitor.visit_Listc                 K  s   |  |jS )zdf.index[4]r   r   r!   r!   r%   visit_Index:  s    zBaseExprVisitor.visit_Indexc           
   	   K  s   ddl m} | |j}| |j}||| j| j| jd}z|j| }W n2 tk
rz   ||| j| j| jd}|| }Y nX | j	|}	| j
|	| jdS )Nr   )eval)Z
local_dictr   r   )r   )pandasr   r   rK   slicer   r   r   AttributeErrorr   r   )
re   rU   r3   Zpd_evalrK   Zslobjresultr   r   rW   r!   r!   r%   visit_Subscript>  s*          zBaseExprVisitor.visit_Subscriptc                 K  sZ   |j }|dk	r| |j}|j}|dk	r4| |j}|j}|dk	rN| |j}t|||S )zdf.index[slice(4,6)]N)lowerr   rK   upperstepr   )re   rU   r3   r   r   r   r!   r!   r%   visit_SliceR  s    zBaseExprVisitor.visit_Slicec                 K  s   t |jdkrtdt|jd tjs0td| jjdkrDtdz| j	|jd f|}W n  t
k
r|   |jd j}Y nX t|d|| _| jdkrtd| j	|jf|S )	z
        support a single assignment node, like

        c = a + b

        set the assigner at the top level, must be a Name node which
        might or might not exist in the resolvers

        r:   z#can only assign a single expressionr   z5left hand side of an assignment must be a single nameNz%cannot assign without a target objectrW   z@left hand side of an assignment must be a single resolvable name)r;   targetsr   rJ   rR   Namer   target
ValueErrorr   r
   r   rV   r   rK   )re   rU   r3   r   r!   r!   r%   visit_Assign`  s     

zBaseExprVisitor.visit_Assignc           	      K  s   |j }|j}|j}t|tjr| |j}z&t||}| j	|}| 
|| jW S  tk
r   t|tjr~|j|kr~| Y S  Y nX tdt|j d S )NzInvalid Attribute context )attrrK   ctxrJ   rR   Loadr   rV   r   r   r   r   r   r   r   rP   rX   )	re   rU   r3   r   rK   r   Zresolvedr   rW   r!   r!   r%   visit_Attribute~  s    
zBaseExprVisitor.visit_Attributec                   s|  t |jtjr(|jjdkr( |j}nft |jtjs@tdnNz |j}W n< t	k
r   zt
|jj}W n tk
r    Y nX Y nX |d krtd|jj t|dr|j}t |t
r fdd|jD }|jrtd|j d|| S  fd	d|jD }|jD ]D}t |tjs6td
|jj d|jr |jj||j< q j|||} j| jdS d S )N__call__z"Only named functions are supportedzInvalid function call rK   c                   s   g | ]}  |qS r!   )r   rB   argr   r!   r%   r     s     z.BaseExprVisitor.visit_Call.<locals>.<listcomp>z
Function "z$" does not support keyword argumentsc                   s   g | ]}  |jqS r!   r   r   r   r!   r%   r     s     z keyword error in function call 'rc   )rW   r   )rJ   funcrR   	Attributer   r   r   r   r   r
   r   r   r   r   rK   r2   keywordsrW   keywordr   r   r   r   )re   rU   r   r3   r   new_argskeyrW   r!   r   r%   
visit_Call  sD    



zBaseExprVisitor.visit_Callc                 C  s   |S r1   r!   )re   rx   r!   r!   r%   translate_In  s    zBaseExprVisitor.translate_Inc              	   K  s   |j }|j}t|dkrF| |d }tj||j|d d}| |S |j}g }t||D ]6\}}	| tj	|	g|| |gd}
|	}|
|
 qZ| tjt |dS )Nr:   r   )rx   r   r   )comparatorsr   rw   )rx   values)rw   r   r;   r   rR   r   r   r   zipZCompareappendBoolOpr   )re   rU   r3   rw   compsrx   r   r   r   compZnew_noder!   r!   r%   visit_Compare  s    
zBaseExprVisitor.visit_Comparec                 C  s   t |ttfr|S | |S r1   )rJ   r   r   r   )re   Zbopr!   r!   r%   _try_visit_binop  s    z BaseExprVisitor._try_visit_binopc                   s    fdd} j }t||S )Nc                   s<    | } |} ||\}}}}| j||S r1   )r   r   r   rx   )rC   yr   r   rx   rr   rU   re   r!   r%   r     s    

z-BaseExprVisitor.visit_BoolOp.<locals>.visitor)r   r   )re   rU   r3   r   Zoperandsr!   r   r%   visit_BoolOp  s    zBaseExprVisitor.visit_BoolOp)NN)r   r   )N)9rX   
__module____qualname____doc__r   r}   __annotations__r   r   r   r   r   r   Zbinary_op_nodesdictr   Zbinary_op_nodes_mapr   Z	unary_opsZunary_op_nodesZunary_op_nodes_maprR   r~   r   r   r   r   rI   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zvisit_Tupler   r   r   r   r   r   r   r   r   r   r!   r!   r!   r%   r|   U  sd   
    
  
'
6r|   Dictr   r   r   r   Tuplec                      s6   e Zd Zeeeeeedfdd fddZ	  Z
S )PandasExprVisitorrD   r   r   c                   s   t  |||| d S r1   superr   r   	__class__r!   r%   r     s    
zPandasExprVisitor.__init__)rX   r   r   r   rI   r>   r0   r.   r   r   __classcell__r!   r!   r  r%   r    s
   

r  r   c                      s,   e Zd Zdddfdd fddZ  ZS )	PythonExprVisitorNc                 C  s   | S r1   r!   rH   r!   r!   r%   r7   
  r8   zPythonExprVisitor.<lambda>r   r   c                   s   t  j||||d d S )N)r   r  r   r  r!   r%   r   	  s    zPythonExprVisitor.__init__)N)rX   r   r   r   r  r!   r!   r  r%   r	    s   r	  c                   @  s   e Zd ZU dZded< ded< ded< dddddddddZedd Zdd ZddddZ	ddddZ
dd Zedd Zd	S )r^   a  
    Object encapsulating an expression.

    Parameters
    ----------
    expr : str
    engine : str, optional, default 'numexpr'
    parser : str, optional, default 'pandas'
    env : Scope, optional, default None
    level : int, optional, default 2
    r   r   r?   r   r   numexprr   Nr   zScope | Noneintr   )r   r   r   levelr   c                 C  sL   || _ |pt|d d| _|| _|| _t| | j| j| j| _|  | _d S )Nr:   )r  )	r   r   r   r   r   PARSERS_visitorr   terms)re   r   r   r   r   r  r!   r!   r%   r      s    zExpr.__init__c                 C  s   t | jdd S )Nr   )rV   r  r   r!   r!   r%   r   /  s    zExpr.assignerc                 C  s   |  | jS r1   )r  r   r   r!   r!   r%   r   3  s    zExpr.__call__r   c                 C  s   t | jS r1   )printingZpprint_thingr  r   r!   r!   r%   __repr__6  s    zExpr.__repr__c                 C  s
   t | jS r1   )r;   r   r   r!   r!   r%   __len__9  s    zExpr.__len__c                 C  s   | j | jS )z&
        Parse an expression.
        )r  r   r   r   r!   r!   r%   r   <  s    z
Expr.parsec                 C  s2   t | jrt| jjgS tdd t| jD S )z1
        Get the names in an expression.
        c                 s  s   | ]}|j V  qd S r1   )rW   )rB   Ztermr!   r!   r%   rE   I  s     zExpr.names.<locals>.<genexpr>)r   r  r[   rW   comflattenr   r!   r!   r%   namesB  s    
z
Expr.names)r
  r   Nr   )rX   r   r   r   r   r   propertyr   r   r  r  r   r  r!   r!   r!   r%   r^     s"   
    
)pythonr   )qr   
__future__r   rR   	functoolsr   r   r   r   r+   typingr   r   Znumpyr   Zpandas.compatr	   Zpandas.errorsr
   Zpandas.core.commoncorecommonr  Zpandas.core.computation.opsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zpandas.core.computation.parsingr   r   Zpandas.core.computation.scoper   Zpandas.io.formats.printingioformatsr  r&   r.   r0   r9   r>   rI   rO   listr   r?   r   r[   dirZ
_all_nodesr\   r   Z_all_node_namesmodZ
_mod_nodesZstmtZ_stmt_nodesr   Z_expr_nodesZexpr_contextZ_expr_context_nodesZboolopZ_boolop_nodesoperatorZ_operator_nodesZunaryopZ_unary_op_nodesZcmpopZ_cmp_op_nodesZcomprehensionZ_comprehension_nodesZexcepthandlerZ_handler_nodes	argumentsZ_arguments_nodesZ_keyword_nodesaliasZ_alias_nodesr   Z_slice_nodesZ_hacked_nodesZ_unsupported_expr_nodesZ_unsupported_nodesZ_base_supported_nodesintersection_msgr<   rg   rh   rp   rt   Z_op_classesr{   ZNodeVisitorr|   Z_python_not_supportedZ_numexpr_supported_callsr  r	  r^   r  r!   r!   r!   r%   <module>   s   D
   "















   =