U
    .e                    @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZmZmZ dd	lmZ dd
lmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' d dlm(Z( d dl)m*Z*m+Z+ ddlm,Z,m-Z-m.Z. dZ/ej0d  dkZ1e2 dkrRddl3m4Z5 eZ6nd dlm7Z5 ej6Z6z
ej8Z8W n e9k
r   dZ8Y nX ej0dkrej:Z;nej;Z;d Z<dZ=dZ>d Z?dZ@dZAdZBdZCd ZDdZEdZFeGeddZHdZIeGedd ZDdZJdZKeL ZMejNZNdd ZOd d! ZPd"d# ZQd$d% ZRdHd&d'ZSG d(d) d)e;ZTG d*d+ d+eUZVG d,d- d-eUZWd.d/ ZXG d0d1 d1ZYG d2d3 d3eZZG d4d5 d5eZZ[G d6d7 d7eZZ\G d8d9 d9eZZ]G d:d; d;eZZ^G d<d= d=Z_G d>d? d?Z`G d@dA dAe`ZaG dBdC dCZbG dDdE dEebZcG dFdG dGe_ZddS )I    N)deque)partial   )	cpu_countget_context)util)TERM_SIGNALhuman_statuspickle_loadsreset_signalsrestart_state)	get_errnomem_rsssend_offset)ExceptionInfo)DummyProcess)CoroStopRestartFreqExceededSoftTimeLimitExceeded
TerminatedTimeLimitExceededTimeoutErrorWorkerLostError	monotonicQueueEmpty)FinalizedebugwarningzEchild process exiting after exceeding memory limit ({0}KiB / {1}KiB)
   Windows)kill_processtree)killg    _B)r!   r!            SIGUSR1g      $@EX_OKi,  皙?c                 C   s>   z
| j }W n tk
r"   d }Y nX |d kr:tt |  S |S N)r   AttributeErrorr   fileno)
connectionZnative r/   1/tmp/pip-unpacked-wheel-39f38r4u/billiard/pool.py_get_send_offsetx   s    

r1   c                 C   s   t t|  S r+   )listmapargsr/   r/   r0   mapstar   s    r6   c                 C   s   t t| d | d S )Nr   r   )r2   	itertoolsstarmapr4   r/   r/   r0   starmapstar   s    r9   c                 O   s   t  j| f|| d S r+   )r   Z
get_loggererror)msgr5   kwargsr/   r/   r0   r:      s    r:   c                 C   s   | t  k	r| | d S r+   )	threadingcurrent_threadstop)threadtimeoutr/   r/   r0   stop_if_not_current   s    rB   c                   @   sb   e Zd ZdZdd Zer<dddZdd	 Zd
d Zdd Z	n"dddZdd	 Zdd Zdd Z	dS )LaxBoundedSemaphorez^Semaphore that checks that # release is <= # acquires,
    but ignores if # releases >= value.c                 C   s   |  j d8  _ |   d S Nr   )_initial_valueacquireselfr/   r/   r0   shrink   s    zLaxBoundedSemaphore.shrinkr   Nc                 C   s   t | | || _d S r+   
_Semaphore__init__rE   rH   valueverboser/   r/   r0   rL      s    zLaxBoundedSemaphore.__init__c              	   C   s<   | j , |  jd7  _|  jd7  _| j   W 5 Q R X d S rD   )_condrE   _valuenotifyrG   r/   r/   r0   grow   s    zLaxBoundedSemaphore.growc              	   C   s<   | j }|( | j| jk r.|  jd7  _|  W 5 Q R X d S rD   )rP   rQ   rE   
notify_allrH   Zcondr/   r/   r0   release   s
    zLaxBoundedSemaphore.releasec                 C   s   | j | jk rt|  q d S r+   )rQ   rE   rK   rV   rG   r/   r/   r0   clear   s    zLaxBoundedSemaphore.clearc                 C   s   t | || || _d S r+   rJ   rM   r/   r/   r0   rL      s    c              	   C   s>   | j }|* |  jd7  _|  jd7  _|  W 5 Q R X d S rD   )_Semaphore__condrE   _Semaphore__valuerR   rU   r/   r/   r0   rS      s
    c              	   C   s<   | j }|( | j| jk r.|  jd7  _|  W 5 Q R X d S rD   )rX   rY   rE   	notifyAllrU   r/   r/   r0   rV      s
    c                 C   s   | j | jk rt|  q d S r+   )rY   rE   rK   rV   rG   r/   r/   r0   rW      s    )r   N)r   N)
__name__
__module____qualname____doc__rI   PY3rL   rS   rV   rW   r/   r/   r/   r0   rC      s   


rC   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.c                    s*   t || _t || _t | j| j d S r+   )reprexcrN   superrL   )rH   rb   rN   	__class__r/   r0   rL      s    

zMaybeEncodingError.__init__c                 C   s   d| j jt| f S )Nz<%s: %s>)re   r[   strrG   r/   r/   r0   __repr__   s    zMaybeEncodingError.__repr__c                 C   s   d| j | jf S )Nz)Error sending result: '%r'. Reason: '%r'.)rN   rb   rG   r/   r/   r0   __str__   s     zMaybeEncodingError.__str__)r[   r\   r]   r^   rL   rg   rh   __classcell__r/   r/   rd   r0   r`      s   r`   c                   @   s   e Zd ZdZdS )WorkersJoinedzAll workers have terminated.N)r[   r\   r]   r^   r/   r/   r/   r0   rj      s   rj   c                 C   s
   t  d S r+   )r   )signumframer/   r/   r0   soft_timeout_sighandler   s    rm   c                
   @   s   e Zd ZdddZdd Zdd	 Zd
d ZdddZdd Zdd Z	e
edfddZdd Zdd Zdd ZefddZdd ZdS ) WorkerNr/   Tc                 C   sz   |d ks t |tkr|dks t|| _|| _|| _|| _|| _|| _|	| _	|||  | _
| _| _|
| _|| _| |  d S Nr   )typeintAssertionErrorinitializerinitargsmaxtasksmax_memory_per_child	_shutdownon_exitsigprotectioninqoutqsynqwrap_exceptionon_ready_countercontribute_to_object)rH   rz   r{   r|   rs   rt   ru   sentinelrx   ry   r}   rv   r~   r/   r/   r0   rL      s     zWorker.__init__c                 C   s   | j | j| j  |_ |_|_| j j |_| jj |_| jrj| jj |_| jj |_	t
| jj|_nd  |_ |_	|_| j jj|_| jjj|_t
| j j|_|S r+   )rz   r{   r|   _writerr-   inqW_fd_readerZoutqR_fdZsynqR_fdsynqW_fdr1   Zsend_syn_offsetZ_send_syn_offsetsend
_quick_putrecv
_quick_getZsend_job_offset)rH   objr/   r/   r0   r      s    zWorker.contribute_to_objectc                 C   s6   | j | j| j| j| j| j| j| j| j| j	| j
| jffS r+   )re   rz   r{   r|   rs   rt   ru   rw   rx   ry   r}   rv   rG   r/   r/   r0   
__reduce__  s            zWorker.__reduce__c                    s   t j d gd fdd	}|t _t }|   |   | j|d z`zt | j|d W nD t	k
r } z&t
d| |dd | |d | W 5 d }~X Y nX W 5 | |d d  X d S )	Nc                    s   | d<  | S ro   r/   )status_exitZ	_exitcoder/   r0   exit  s    zWorker.__call__.<locals>.exitpidr   zPool process %r error: %rr   exc_info)N)sysr   osgetpid_make_child_methods
after_forkon_loop_start_do_exitworkloop	Exceptionr:   )rH   r   r   rb   r/   r   r0   __call__  s    (zWorker.__call__c              	   C   st   |d kr|rt nt}| jd k	r*| || tjdkrfz"| jt	||ff t
d W 5 t| X n
t| d S )Nwin32r   )
EX_FAILUREr)   rx   r   platformr   r   r{   putDEATHtimesleep)rH   r   exitcoderb   r/   r/   r0   r   *  s    

zWorker._do_exitc                 C   s   d S r+   r/   rH   r   r/   r/   r0   r   :  s    zWorker.on_loop_startc                 C   s   |S r+   r/   )rH   resultr/   r/   r0   prepare_result=  s    zWorker.prepare_resultc              
      s  |p
t  }jj}j}j}j}jp.d}j}	j	}
j
  fdd}d}z|d ksp|r||k r|
 }|rX|\}}|tkst|\}}}}}|t||| ||ff  r||}|sqXzd|	|||f}W n tk
r   dt f}Y nX z|t||||ff W np tk
r } zPt \}}}z8t||d }tt||f}|t||d|f|ff W 5 ~X W 5 d }~X Y nX |d7 }|dkrXt }|dkrtd |dkrX||krXtt|| tW 2S qX|d	| |r||krtntW 
S tW S j|d X d S )
Nr   c                    s`   d}|dkr$t d| jj dd   }|rR|\}}|tkrBdS |tksNtdS |d7 }qd S )Nr   <   z(!!!WAIT FOR ACK TIMEOUT: job:%r fd:%r!!!r   r   FT)r:   r|   r   r-   NACKACKrr   )Zjidireqtype_r5   Z_wait_for_synrH   r/   r0   wait_for_synL  s     
 z%Worker.workloop.<locals>.wait_for_syn)	completedTFr   z'worker unable to determine memory usagezworker exiting after %d tasks)r   r   r{   r   r   r   ru   rv   r   wait_for_jobr   _ensure_messages_consumedTASKrr   r   r   r   READYr   r   r`   r   r:   r    MAXMEM_USED_FMTformat
EX_RECYCLEr   r)   )rH   r   nowr   r   r   r   ru   rv   r   r   r   r   r   r   Zargs_jobr   funr5   r<   Zconfirmr   rb   _tbwrappedeinfoZused_kbr/   r   r0   r   @  sp    
  
 

zWorker.workloopc                 C   sJ   | j s
dS ttD ]*}| j j|kr2td|  dS tt qtd dS )zr Returns true if all messages sent out have been received and
        consumed within a reasonable amount of time Fz*ensured messages consumed after %d retriesTz<could not ensure all messages were consumed prior to exiting)	r~   range)GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMITrN   r   r   r   ,GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVALr    )rH   r   retryr/   r/   r0   r     s    
z Worker._ensure_messages_consumedc                 C   s   t | jdr| jj  t | jdr0| jj  | jd k	rF| j| j  t| j	d t
d k	rftt
t zttjtj W n tk
r   Y nX d S )Nr   r   )full)hasattrrz   r   closer{   r   rs   rt   r   ry   SIG_SOFT_TIMEOUTsignalrm   SIGINTSIG_IGNr,   rG   r/   r/   r0   r     s    
zWorker.after_forkc                    s`   |j t|drP|jj t|dr@|jr@|jtffdd	}q\ fdd}nfdd}|S )Nr   get_payloadc                    s   d|  fS NTr/   )rA   loads)r   r/   r0   _recv  s    z'Worker._make_recv_method.<locals>._recvc                    s    | rd fS dS NTFNr/   rA   )_pollgetr/   r0   r     s    
c                    s.   zd | dfW S  t jk
r(   Y dS X d S NTr   r   r   r   )r   r/   r0   r     s    )r   r   r   pollr   r
   )rH   connr   r/   )r   r   r   r0   _make_recv_method  s    
zWorker._make_recv_methodc                 C   s*   |  | j| _| jr |  | jnd | _d S r+   )_make_protected_receiverz   r   r|   r   )rH   r   r/   r/   r0   r     s
    zWorker._make_child_methodsc                    s2   |  | | jr| jjnd tf fdd	}|S )Nc              
      s   r r| d t tz d\}}|s2W d S W nV ttfk
r } z4t|tjkrbW Y  d S | dt|j t t	W 5 d }~X Y nX |d kr| d t t	|S )Nzworker got sentinel -- exiting      ?zworker got %s -- exiting)

SystemExitr)   EOFErrorIOErrorr   errnoZEINTRrp   r[   r   )r   readyr   rb   Z_receiveZshould_shutdownr/   r0   receive  s     


z/Worker._make_protected_receive.<locals>.receive)r   rw   is_setr   )rH   r   r   r/   r   r0   r     s    
zWorker._make_protected_receive)
NNr/   NNNTTNN)N)r[   r\   r]   rL   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   r   r/   r/   r/   r0   rn      s*               

Mrn   c                       sN   e Zd Zdd Zdd Z fddZdd Zdd
dZdd Zdd Z	  Z
S )
PoolThreadc                 O   s    t |  t| _d| _d| _d S NFT)r   rL   RUN_state_was_starteddaemonrH   r5   r<   r/   r/   r0   rL     s    
zPoolThread.__init__c              
   C   s   z
|   W S  tk
rX } z0tdt| j|dd tt t t	
  W 5 d }~X Y nB tk
r } z$tdt| j|dd td W 5 d }~X Y nX d S )NzThread %r crashed: %rr   r   )bodyr   r:   rp   r[   _killr   r   r   r   r   r   r   rH   rb   r/   r/   r0   run  s    
zPoolThread.runc                    s   d| _ tt| j|| d S r   )r   rc   r   startr   rd   r/   r0   r     s    zPoolThread.startc                 C   s   d S r+   r/   rG   r/   r/   r0   on_stop_not_started  s    zPoolThread.on_stop_not_startedNc                 C   s    | j r| | d S |   d S r+   )r   joinr   rH   rA   r/   r/   r0   r?   	  s    
zPoolThread.stopc                 C   s
   t | _d S r+   )	TERMINATEr   rG   r/   r/   r0   	terminate  s    zPoolThread.terminatec                 C   s
   t | _d S r+   )CLOSEr   rG   r/   r/   r0   r     s    zPoolThread.close)N)r[   r\   r]   rL   r   r   r   r?   r   r   ri   r/   r/   rd   r0   r     s   
r   c                       s$   e Zd Z fddZdd Z  ZS )
Supervisorc                    s   || _ t   d S r+   )poolrc   rL   )rH   r   rd   r/   r0   rL     s    zSupervisor.__init__c                 C   s   t d td | j}z~|j}td|j d|_tdD ]*}| jtkr:|jtkr:|	  td q:||_| jtkr|jtkr|	  td qlW n& t
k
r   |  |   Y nX t d d S )Nzworker handler startingg?
   r   r*   zworker handler exiting)r   r   r   r   r   
_processesr   r   r   _maintain_poolr   r   r   )rH   r   Z
prev_stater   r/   r/   r0   r     s&    
zSupervisor.body)r[   r\   r]   rL   r   ri   r/   r/   rd   r0   r     s   r   c                       s4   e Zd Z fddZdd Zdd Zdd Z  ZS )	TaskHandlerc                    s,   || _ || _|| _|| _|| _t   d S r+   )	taskqueuer   outqueuer   cacherc   rL   )rH   r   r   r   r   r   rd   r/   r0   rL   =  s    zTaskHandler.__init__c           
      C   s  | j }| j}| j}t|jd D ]P\}}d }d}zt|D ]\}}| jrTtd  qz|| W q: tk
r   td Y  qY q: t	k
r   |d d \}}	z|| 
|	dt f W n tk
r   Y nX Y q:X q:|rtd ||d  W qW  qzW q t	k
rn   |r"|d d nd\}}	||krN|| 
|	d dt f |rjtd ||d  Y qX qtd	 |   d S )
Nz'task handler found thread._state != RUNzcould not put task on queuer%   Fzdoing set_length()r   )r   r   ztask handler got sentinel)r   r   r   iterr   	enumerater   r   r   r   _setr   KeyErrorr   tell_others)
rH   r   r   r   ZtaskseqZ
set_lengthtaskr   r   indr/   r/   r0   r   E  sH    


zTaskHandler.bodyc                 C   sl   | j }| j}| j}z0td |d  td |D ]}|d  q2W n tk
r^   td Y nX td d S )Nz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got IOError when sending sentinelsztask handler exiting)r   r   r   r   r   )rH   r   r   r   pr/   r/   r0   r  o  s    
zTaskHandler.tell_othersc                 C   s   |    d S r+   )r  rG   r/   r/   r0   r     s    zTaskHandler.on_stop_not_started)r[   r\   r]   rL   r   r  r   ri   r/   r/   rd   r0   r   ;  s   *r   c                       sT   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
  ZS )TimeoutHandlerc                    s,   || _ || _|| _|| _d | _t   d S r+   )	processesr   t_softt_hard_itrc   rL   )rH   r  r   r  r  rd   r/   r0   rL     s    zTimeoutHandler.__init__c                    s   t  fddt| jD dS )Nc                 3   s$   | ]\}}|j  kr||fV  qd S r+   r   .0r   procr   r/   r0   	<genexpr>  s   
z1TimeoutHandler._process_by_pid.<locals>.<genexpr>NN)nextr   r  r   r/   r   r0   _process_by_pid  s
    zTimeoutHandler._process_by_pidc              
   C   sv   t d| | |j\}}|s"d S |jdd zt|jt W n2 tk
rp } zt|tj	kr` W 5 d }~X Y nX d S )Nzsoft time limit exceeded for %rTsoft)
r   r  _worker_pidhandle_timeoutr   r   OSErrorr   r   ESRCH)rH   r   process_indexrb   r/   r/   r0   on_soft_timeout  s    
zTimeoutHandler.on_soft_timeoutc                 C   s|   |  rd S td| zt|jW n( tk
rL   ||jdt f Y nX | |j\}}|j	dd |rx| 
| d S )Nzhard time limit exceeded for %rFr  )r   r   r   _timeoutr   _jobr   r  r  r  _trywaitkill)rH   r   r  r  r/   r/   r0   on_hard_timeout  s    
zTimeoutHandler.on_hard_timeoutc                 C   s   t d|j zBt|j|jkrDt d|j tt|jtj n|  W n t	k
rb   Y nX |j
jddrvd S t d|j zFt|j|jkrt d|j tt|jtj nt|jt W n t	k
r   Y nX d S )Nztimeout: sending TERM to %szIworker %s is a group leader. It is safe to kill (SIGTERM) the whole groupr*   r   z/timeout: TERM timed-out, now sending KILL to %szIworker %s is a group leader. It is safe to kill (SIGKILL) the whole group)r   _namer   getpgidr   killpgr   SIGTERMr   r  _popenwaitSIGKILLr   rH   workerr/   r/   r0   r    s$    zTimeoutHandler._trywaitkillc                 #   s   | j | j }}t }| j}| j}dd }| jtkrt| j |rXt fdd|D } 	 D ]j\}}|j
}	|j}
|
d kr|}
|j}|d kr|}||	|r|| q`||kr`||	|
r`|| || q`d V  q(d S )Nc                 S   s"   | r|sdS t  | | krdS d S r   r   )r   rA   r/   r/   r0   
_timed_out  s    z2TimeoutHandler.handle_timeouts.<locals>._timed_outc                 3   s   | ]}| kr|V  qd S r+   r/   )r  kr   r/   r0   r    s      z1TimeoutHandler.handle_timeouts.<locals>.<genexpr>)r  r  setr  r  r   r   copyr   items_time_accepted_soft_timeoutr  add)rH   r  r  Zdirtyr  r  r'  r   r   Zack_timesoft_timeoutZhard_timeoutr/   r)  r0   handle_timeouts  s.    


zTimeoutHandler.handle_timeoutsc                 C   sN   | j tkrBz|  D ]}td qW q  tk
r>   Y qBY q X q td d S )Nr   ztimeout handler exiting)r   r   r1  r   r   r   r   rH   r   r/   r/   r0   r     s    
zTimeoutHandler.bodyc                 G   sB   | j d kr|  | _ zt| j  W n tk
r<   d | _ Y nX d S r+   )r	  r1  r  StopIterationrH   r5   r/   r/   r0   handle_event   s    

zTimeoutHandler.handle_event)r[   r\   r]   rL   r  r  r  r  r1  r   r5  ri   r/   r/   rd   r0   r    s   &	r  c                       sT   e Zd Zd fdd	Zdd Zdd Zdd	d
ZdddZdd ZdddZ	  Z
S )ResultHandlerNc                    s^   || _ || _|| _|| _|| _|| _|| _d | _d| _|| _	|	| _
|
| _|   t   d S )NF)r   r   r   r   join_exited_workersputlockr   r	  _shutdown_completecheck_timeoutson_job_readyon_ready_counters_make_methodsrc   rL   )rH   r   r   r   r   r7  r8  r   r:  r;  r<  rd   r/   r0   rL     s    zResultHandler.__init__c                 C   s   | j dd d S )NT)r1  )finish_at_shutdownrG   r/   r/   r0   r     s    z!ResultHandler.on_stop_not_startedc                    sl   j  jjj fdd} fdd}dd }t|t|t|i _fdd}|_d S )	Nc              	      s<   d_ z |  |||| W n ttfk
r6   Y nX d S ro   )R_ackr   r,   )r   r   time_acceptedr   r   )r   r   r/   r0   on_ack'  s
    z+ResultHandler._make_methods.<locals>.on_ackc              	      s   d k	r| ||| z |  }W n t k
r8   Y d S X jrtt| d }|r|jkrj| }|  | jd7  _W 5 Q R X | sd k	r  z|	|| W n t k
r   Y nX d S rD   )
r   r<  r  r   worker_pidsZget_lockrN   r   rV   r   )r   r   r   r   itemZ
worker_pidr~   )r   r;  r8  rH   r/   r0   on_ready/  s&    

z-ResultHandler._make_methods.<locals>.on_readyc              
   S   sH   zt | t W n2 tk
rB } zt|tjkr2 W 5 d }~X Y nX d S r+   )r   r$   r   r  r   r   r  )r   r   rb   r/   r/   r0   on_deathF  s
    z-ResultHandler._make_methods.<locals>.on_deathc                    s>   | \}}z | |  W n  t k
r8   td|| Y nX d S )NzUnknown job state: %s (args=%s))r   r   )r  stater5   )state_handlersr/   r0   on_state_changeQ  s
    z4ResultHandler._make_methods.<locals>.on_state_change)	r   r8  r   r;  r   r   r   rH  rI  )rH   rB  rE  rF  rI  r/   )r   r;  r8  r   rH   rH  r0   r=  !  s          
zResultHandler._make_methodsr   c              
   c   s   | j }| j}z||\}}W n6 ttfk
rR } ztd| t W 5 d }~X Y nX | jrv| jtkshttd t |r|d krtd t || |dkrqnqd V  qd S )N result handler got %r -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelr   )	r   rI  r   r   r   r   r   r   rr   )rH   rA   r   rI  r   r  rb   r/   r/   r0   _process_resultY  s(    
zResultHandler._process_resultc              	   C   sR   | j tkrN| jd kr | d| _zt| j W n ttfk
rL   d | _Y nX d S ro   )r   r   r	  rK  r  r3  r   )rH   r-   eventsr/   r/   r0   r5  t  s    

zResultHandler.handle_eventc                 C   sV   t d z>| jtkrDz| dD ]}q W q
 tk
r@   Y qDY q
X q
W 5 |   X d S )Nzresult handler startingr   )r   r>  r   r   rK  r   r2  r/   r/   r0   r   }  s    
zResultHandler.bodyFc              
   C   s  d| _ | j}| j}| j}| j}| j}| j}| j}d }	|r*| jt	kr*|d k	rT|  z|d\}
}W n6 t
tfk
r } ztd| W Y d S d }~X Y nX |
r|d krtd q4|| z|dd W q4 tk
r&   t }|	s|}	n8||	 dkrtd Y q*tdtt||	 d d	 Y q4X q4t|d
rtd z,tdD ]}|j s^ qh|  qHW n t
tfk
r   Y nX tdt|| j d S )NTr   rJ  z&result handler ignoring extra sentinel)shutdowng      @z!result handler exiting: timed outz6result handler: all workers terminated, timeout in %ssr   r   z"ensuring that outqueue is not fullr   z7result handler exiting: len(cache)=%s, thread._state=%s)r9  r   r   r   r   r7  r:  rI  r   r   r   r   r   rj   r   absminr   r   r   len)rH   r1  r   r   r   r   r7  r:  rI  Ztime_terminater   r  rb   r   r   r/   r/   r0   r>    s^    
 z ResultHandler.finish_at_shutdown)N)r   )NN)F)r[   r\   r]   rL   r   r=  rK  r5  r   r>  ri   r/   r/   rd   r0   r6  	  s    8

	r6  c                   @   s<  e Zd ZdZdZeZeZeZeZe	Z	e
Z
dwddZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdxd!d"Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Z d3d4 Z!dyd5d6Z"dzd7d8Z#d9d: Z$d;d< Z%d=d> Z&d?d@ Z'dAdB Z(dCdD Z)dEdF Z*dGdH Z+dIdJ Z,di fdKdLZ-d{dMdNZ.d|dOdPZ/d}dQdRZ0d~dSdTZ1ddUdVZ2di ddddddddddfdWdXZ3dYdZ Z4dd[d\Z5dd]d^Z6dd_d`Z7e8dadb Z9dcdd Z:dedf Z;dgdh Z<e8didj Z=dkdl Z>dmdn Z?e8dodp Z@eAdqdr ZBeAdsdt ZCeDdudv ZEdS )PoolzS
    Class which supports an async version of applying functions to arguments.
    TNr/   r   Fc                 K   s  |pt  | _|| _|   t | _i | _t| _|| _	|| _
|| _|| _|| _|| _|| _|p^t| _|
| _|| _|| _|| _|| _i | _|| _t|p| j	d k	p| j
d k	| _|rtd krttd d }|d kr|  n|| _ |pt!| j d | _"t#||	pd| _#|d k	rt$|st%d|d k	r:t$|s:t%d| jj&| _'g | _(i | _)i | _*|| _+|pjt,| j | _-t.| j D ]}| /| qx| 0| | _1|r| j12  | 3| j| j4| j5| j(| j| _6|r| j62  d | _7| jr$| 8| j(| j| j
| j	| _9t: | _;d| _<| =  |s6| j9j>| _7nd | _9d| _<d | _;| ? | _@| j@j>| _A|rZ| j@2  tB| | jC| j| jD| j5| j(| j1| j6| j@| j| j9| E f
dd| _Fd S )	NWSoft timeouts are not supported: on this platform: It does not have the SIGUSR1 signal.d   r   zinitializer must be a callablez on_process_exit must be callableF   )r5   Zexitpriority)Gr   _ctxsynack_setup_queuesr   
_taskqueue_cacher   r   rA   r0  _maxtasksperchild_max_memory_per_child_initializer	_initargs_on_process_exitLOST_WORKER_TIMEOUTlost_worker_timeouton_process_upon_process_downon_timeout_seton_timeout_cancelthreadsZreadersallow_restartboolenable_timeoutsr   warningswarnUserWarningr   r   roundmax_restartsr   callable	TypeErrorProcess_Process_pool	_poolctrl_on_ready_countersputlocksrC   _putlockr   _create_worker_processr   _worker_handlerr   r   r   	_outqueue_task_handlerr:  r  _timeout_handlerLock_timeout_handler_mutex_timeout_handler_started_start_timeout_handlerr5  create_result_handler_result_handlerhandle_result_eventr   _terminate_pool_inqueue_help_stuff_finish_args
_terminate)rH   r  rs   rt   ZmaxtasksperchildrA   r0  r`  rm  Zmax_restart_freqra  rb  rc  rd  re  Z	semaphoreru  rf  rV  Zon_process_exitcontextrv   rh  r<   r   r/   r/   r0   rL     s    



  


    zPool.__init__c                 O   s   | j ||S r+   )rq  )rH   r5   kwdsr/   r/   r0   rp  H  s    zPool.Processc                 C   s   | | j|dS )N)target)r   rp  r%  r/   r/   r0   WorkerProcessK  s    zPool.WorkerProcessc              
   K   s:   | j | j| j| j| j| j| j| j| j| j	f	d| j
i|S )Nr<  )r6  ry  r   rY  _poll_result_join_exited_workersrv  r   r:  r;  rt  )rH   extra_kwargsr/   r/   r0   r  N  s         zPool.create_result_handlerc                 C   s   d S r+   r/   )rH   r   r   r   r   r/   r/   r0   r;  W  s    zPool.on_job_readyc                 C   s   | j | j| jfS r+   )r  rz  rr  rG   r/   r/   r0   r  Z  s    zPool._help_stuff_finish_argsc                 C   s$   zt  W S  tk
r   Y dS X d S rD   )r   NotImplementedErrorrG   r/   r/   r0   r   ]  s    zPool.cpu_countc                 G   s   | j j| S r+   )r  r5  r4  r/   r/   r0   r  c  s    zPool.handle_result_eventc                 C   s   d S r+   r/   )rH   r&  Zqueuesr/   r/   r0   _process_register_queuesf  s    zPool._process_register_queuesc                    s   t  fddt| jD dS )Nc                 3   s$   | ]\}}|j  kr||fV  qd S r+   r   r
  r   r/   r0   r  j  s   
z'Pool._process_by_pid.<locals>.<genexpr>r  )r  r   rr  r   r/   r   r0   r  i  s
    zPool._process_by_pidc                 C   s   | j | jd fS r+   )r  ry  rG   r/   r/   r0   get_process_queueso  s    zPool.get_process_queuesc                 C   s   | j r| j nd }|  \}}}| jd}| | j|||| j| j| j	|| j
| j| j| j|d}| j| | ||||f |jdd|_d|_||_|  || j|j< || j|j< | jr| | |S )Nr   )ry   r}   rv   r~   rp  Z
PoolWorkerT)rf  rU  Eventr  Valuer  rn   r\  r]  rZ  r^  re  _wrap_exceptionr[  rr  appendr  namereplacer   indexr   rs  r   rt  ra  )rH   r   r   rz   r{   r|   r~   wr/   r/   r0   rw  r  s8          

zPool._create_worker_processc                 C   s   d S r+   r/   r%  r/   r/   r0   process_flush_queues  s    zPool.process_flush_queuesc                    s|  d}dd t | j D D ]2}|p(t }|j\}}|| |jkr| || q|rdt| jsdt	 i i  }t
tt| jD ]}| j| }|j}	|j}
|
dks|	dk	rtd| |
dk	r|  td| ||j< |	||j< |	ttfkrt|ddstd|j|jt|	d	d
 | | | j|= | j|j= | j|j= qrxdd | jD  t | j D ]}t fdd| D d}|r| || | s6||pd	}	|}|rt|ddr||	 n|  |||	 nJ|j!}|j"}|r|# s| ||j n|rd|# sd| ||j qd D ](}| j$rB|s^| %| | $| qBt | S g S )zCleanup after any worker processes which have exited due to
        reaching their specified lifetime. Returns True if any workers were
        cleaned up.
        Nc                 S   s   g | ]}|  s|jr|qS r/   )r   _worker_lost)r  r   r/   r/   r0   
<listcomp>  s     z-Pool._join_exited_workers.<locals>.<listcomp>z!Supervisor: cleaning up worker %dzSupervisor: worked %d joined_controlled_terminationFz Process %r pid:%r exited with %rr   r   c                 S   s   g | ]
}|j qS r/   r   r  r  r/   r/   r0   r    s     c                 3   s"   | ]}|ks| kr|V  qd S r+   r/   r  r   Zall_pidscleanedr/   r0   r    s     z,Pool._join_exited_workers.<locals>.<genexpr>_job_terminated)&r2   rY  valuesr   r  _lost_worker_timeoutmark_as_worker_lostrP  rr  rj   reversedr   r   r"  r   r   r   r)   r   getattrr:   r  r	   r  rs  rt  r  rC  on_job_process_downr   r   _set_terminatedon_job_process_lost	_write_to_scheduled_for	_is_aliverb  _process_cleanup_queues)rH   rM  r   r   Z	lost_timeZlost_ret	exitcodesr   r&  r   popenZacked_by_goner  Zwrite_toZ	sched_forr/   r  r0   r    s    








  



  
zPool._join_exited_workersc                 C   s   d S r+   r/   )rH   r   r&  r/   r/   r0   on_partial_read  s    zPool.on_partial_readc                 C   s   d S r+   r/   r%  r/   r/   r0   r    s    zPool._process_cleanup_queuesc                 C   s   d S r+   r/   )rH   r   Zpid_goner/   r/   r0   r    s    zPool.on_job_process_downc                 C   s   t  |f|_d S r+   )r   r  )rH   r   r   r   r/   r/   r0   r    s    zPool.on_job_process_lostc                 C   sF   zt dt||jW n& t k
r@   |d dt f Y nX d S )Nz(Worker exited prematurely: {0} Job: {1}.F)r   r   r	   r  r   r   )rH   r   r   r/   r/   r0   r    s     zPool.mark_as_worker_lostc                 C   s   | S r+   r/   rG   r/   r/   r0   	__enter__  s    zPool.__enter__c                 G   s   |   S r+   )r   )rH   r   r/   r/   r0   __exit__  s    zPool.__exit__c                 C   s   d S r+   r/   rH   nr/   r/   r0   on_grow  s    zPool.on_growc                 C   s   d S r+   r/   r  r/   r/   r0   	on_shrink  s    zPool.on_shrinkc                 C   sb   t |  D ]H\}}|  jd8  _| jr2| j  |  | d ||d kr q^qtdd S )Nr   z&Can't shrink pool. All processes busy!)r   _iterinactiver   rv  rI   Zterminate_controlledr  
ValueError)rH   r  r   r&  r/   r/   r0   rI     s    

zPool.shrinkc                 C   s:   t |D ]"}|  jd7  _| jr| j  q| | d S rD   )r   r   rv  rS   r  )rH   r  r   r/   r/   r0   rS     s
    z	Pool.growc                 c   s    | j D ]}| |s|V  qd S r+   )rr  _worker_activer%  r/   r/   r0   r    s    

zPool._iterinactivec                 C   s(   | j  D ]}|j| kr
 dS q
dS )NTF)rY  r  r   rC  )rH   r&  r   r/   r/   r0   r    s    zPool._worker_activec              	   C   s   t | jt| j D ]l}| jtkr( dS z"|rH|| ttfkrH| j	  W n t
k
rh   | j	  Y nX | |   td qdS )zBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        Nzadded worker)r   r   rP  rr  r   r   r)   r   r   step
IndexErrorrw  _avail_indexr   )rH   r  r   r/   r/   r0   _repopulate_pool#  s    
zPool._repopulate_poolc                    sD   t | j| jk sttdd | jD  t fddt| jD S )Nc                 s   s   | ]}|j V  qd S r+   )r  )r  r  r/   r/   r0   r  4  s     z$Pool._avail_index.<locals>.<genexpr>c                 3   s   | ]}| kr|V  qd S r+   r/   )r  r   indicesr/   r0   r  5  s      )rP  rr  r   rr   r*  r  r   rG   r/   r  r0   r  2  s    zPool._avail_indexc                 C   s
   |    S r+   )r  rG   r/   r/   r0   did_start_ok7  s    zPool.did_start_okc                 C   s<   |   }| | tt|D ]}| jdk	r| j  qdS )zF"Clean up any exited workers and start replacements for them.
        N)r  r  r   rP  rv  rV   )rH   Zjoinedr   r/   r/   r0   r   :  s
    

zPool._maintain_poolc              
   C   s   | j jtkr| jtkrz|   W n\ tk
rH   |   |    Y n8 tk
r~ } zt|t	j
krlt| W 5 d }~X Y nX d S r+   )rx  r   r   r   r   r   r   r  r   r   ZENOMEMMemoryErrorr   r/   r/   r0   maintain_poolC  s    zPool.maintain_poolc                    sF    j   _ j   _ jjj _ jjj _	 fdd}| _
d S )Nc                    s    j j| rd  fS dS r   )ry  r   r   r   r   rG   r/   r0   r  V  s    z(Pool._setup_queues.<locals>._poll_result)rU  SimpleQueuer  ry  r   r   r   r   r   r   r  rH   r  r/   rG   r0   rW  P  s    zPool._setup_queuesc              	   C   s<   | j r8| jd k	r8| j | js.d| _| j  W 5 Q R X d S r   )re  r{  r}  r~  r   rG   r/   r/   r0   r  \  s
    zPool._start_timeout_handlerc                 C   s    | j tkr| ||| S dS )z8
        Equivalent of `func(*args, **kwargs)`.
        N)r   r   apply_asyncr   )rH   funcr5   r  r/   r/   r0   applye  s    
z
Pool.applyc                 C   s"   | j tkr| ||t| S dS )z
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        N)r   r   
_map_asyncr9   r   rH   r  iterable	chunksizer/   r/   r0   r8   l  s
    
 zPool.starmapc                 C   s"   | j tkr| ||t|||S dS )z=
        Asynchronous version of `starmap()` method.
        N)r   r   r  r9   rH   r  r  r  callbackerror_callbackr/   r/   r0   starmap_asyncv  s
    
 zPool.starmap_asyncc                 C   s    | j tkr| ||| S dS )zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        N)r   r   	map_asyncr   r  r/   r/   r0   r3     s    
zPool.mapc                    s   | j tkrdS |p| j}|dkrXt| j|d| j fddt|D jf S |dksdt	t
 ||}t| j|d| jfddt|D jf dd D S dS )zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        Nr   r`  c                 3   s(   | ] \}}t j| |fi ffV  qd S r+   r   r  r  r   xr  r   r/   r0   r    s   zPool.imap.<locals>.<genexpr>c                 3   s(   | ] \}}t  j|t|fi ffV  qd S r+   r   r  r6   r  r   r/   r0   r    s   c                 s   s   | ]}|D ]
}|V  q
qd S r+   r/   r  chunkrD  r/   r/   r0   r    s       )r   r   r`  IMapIteratorrY  rX  r   r   _set_lengthrr   rQ  
_get_tasksrH   r  r  r  r`  task_batchesr/   r  r0   imap  s4    


z	Pool.imapc                    s   | j tkrdS |p| j}|dkrXt| j|d| j fddt|D jf S |dksdt	t
 ||}t| j|d| jfddt|D jf dd D S dS )zL
        Like `imap()` method but ordering of results is arbitrary.
        Nr   r  c                 3   s(   | ] \}}t j| |fi ffV  qd S r+   r  r  r  r/   r0   r    s   z&Pool.imap_unordered.<locals>.<genexpr>c                 3   s(   | ] \}}t  j|t|fi ffV  qd S r+   r  r  r  r/   r0   r    s   c                 s   s   | ]}|D ]
}|V  q
qd S r+   r/   r  r/   r/   r0   r    s       )r   r   r`  IMapUnorderedIteratorrY  rX  r   r   r  rr   rQ  r  r  r/   r  r0   imap_unordered  s8    

  
zPool.imap_unorderedc                 C   s  | j tkrdS |	p| j}	|
p | j}
|p*| j}|	rJtdkrJttd d}	| j tkr|dkrd| j	n|}|r| j
dk	r| j
  t| j|||||	|
|| j| j|| jr| jnd|d}|
s|	r|   | jr| jt|jd|||ffgdf n| t|jd|||ff |S dS )a  
        Asynchronous equivalent of `apply()` method.

        Callback is called when the functions return value is ready.
        The accept callback is called when the job is accepted to be executed.

        Simplified the flow is like this:

            >>> def apply_async(func, args, kwds, callback, accept_callback):
            ...     if accept_callback:
            ...         accept_callback()
            ...     retval = func(*args, **kwds)
            ...     if callback:
            ...         callback(retval)

        NrR  )rc  rd  callbacks_propagatesend_ackcorrelation_id)r   r   r0  rA   r`  r   ri  rj  rk  ru  rv  rF   ApplyResultrY  rc  rd  rV  r  r  re  rX  r   r   r  r   )rH   r  r5   r  r  r  accept_callbacktimeout_callbackZwaitforslotr0  rA   r`  r  r  r   r/   r/   r0   r    sR    




      	  zPool.apply_asyncc                 C   s   d S r+   r/   )rH   responser   r   fdr/   r/   r0   r    s    zPool.send_ackc              
   C   sl   |  |\}}|d k	rhzt||p"t W n2 tk
rZ } zt|tjkrJ W 5 d }~X Y nX d|_d|_d S r   )	r  r   r   r  r   r   r  r  r  )rH   r   sigr  r   rb   r/   r/   r0   terminate_job  s    zPool.terminate_jobc                 C   s   |  ||t|||S )z<
        Asynchronous equivalent of `map()` method.
        )r  r6   r  r/   r/   r0   r  	  s         zPool.map_asyncc           	         s   | j tkrdS t|ds t|}|dkrPtt|t| jd \}}|rP|d7 }t|dkr`d}t|||}t	| j
|t|||d| j fddt|D df S )	zY
        Helper function to implement map, starmap and their async counterparts.
        N__len__r&   r   r   r  c                 3   s(   | ] \}}t j| |fi ffV  qd S r+   r  r  mapperr   r/   r0   r  &  s   z"Pool._map_async.<locals>.<genexpr>)r   r   r   r2   divmodrP  rr  rQ  r  	MapResultrY  rX  r   r   )	rH   r  r  r  r  r  r  extrar  r/   r  r0   r    s(    

zPool._map_asyncc                 c   s0   t |}tt||}|s d S | |fV  qd S r+   )r   tupler7   islice)r  itsizer  r/   r/   r0   r  *  s
    zPool._get_tasksc                 C   s   t dd S )Nz:pool objects cannot be passed between processes or pickled)r  rG   r/   r/   r0   r   3  s    zPool.__reduce__c                 C   sL   t d | jtkrHt| _| jr(| j  | j  | j	d  t
| j d S )Nzclosing pool)r   r   r   r   rv  rW   rx  r   rX  r   rB   rG   r/   r/   r0   r   8  s    


z
Pool.closec                 C   s$   t d t| _| j  |   d S )Nzterminating pool)r   r   r   rx  r   r  rG   r/   r/   r0   r   B  s    
zPool.terminatec                 C   s   t |  d S r+   )rB   )task_handlerr/   r/   r0   _stop_task_handlerH  s    zPool._stop_task_handlerc                 C   s   | j ttfksttd t| j td | | j td t| j	 td t
| jD ]2\}}td|d t| j| |jd k	r\|  q\td d S )Nzjoining worker handlerjoining task handlerjoining result handlerzresult handler joinedzjoining worker %s/%s (%r)r   zpool join complete)r   r   r   rr   r   rB   rx  r  rz  r  r   rr  rP  r"  r   )rH   r   r  r/   r/   r0   r   L  s    



z	Pool.joinc                 C   s   | j  D ]}|  q
d S r+   )rs  r  r*  )rH   er/   r/   r0   restart[  s    zPool.restartc                 C   s>   t d | j  | r:| j r:| j  td qd S )Nz7removing tasks from inqueue until task handler finishedr   )	r   Z_rlockrF   is_aliver   r   r   r   r   )inqueuer  rr  r/   r/   r0   _help_stuff_finish_  s
    

zPool._help_stuff_finishc                 C   s   | d  d S r+   )r   )clsr   r   r/   r/   r0   _set_result_sentinelh  s    zPool._set_result_sentinelc                 C   s>  t d |  |  |d  t d | j|
  |  | || |	d k	rX|	  |rt|d drt d |D ]}| rv|  qvt d | | t d |  |	d k	rt d |	t	 |rt|d drt d	 |D ]*}|
 rt d
|j |jd k	r|  qt d |r,|  |r:|  d S )Nzfinalizing poolz&helping task handler/workers to finishr   r   zterminating workersr  r  zjoining timeout handlerzjoining pool workerszcleaning up worker %dzpool workers joined)r   r   r   r  r  r   r  r  r?   TIMEOUT_MAXr  r   r"  r   r   )r   r   r  r   r   Zworker_handlerr  Zresult_handlerr   Ztimeout_handlerZhelp_stuff_finish_argsr  r/   r/   r0   r  l  sD    






zPool._terminate_poolc                 C   s   dd | j D S )Nc                 S   s   g | ]}|j jqS r/   )r"  r   r  r/   r/   r0   r    s     z*Pool.process_sentinels.<locals>.<listcomp>)rr  rG   r/   r/   r0   process_sentinels  s    zPool.process_sentinels)NNr/   NNNNNr   NNNNTNFFFNNNF)F)r   )r   )N)NNN)N)r   N)r   N)N)NNN)NNN)Fr[   r\   r]   r^   r  rn   r   r   r  r6  r   rL   rp  r  r  r;  r  r   r  r  r  r  rw  r  r  r  r  r  r  r  r  r  r  r  rI   rS   r  r  r  r  r  r   r  rW  r  r  r8   r  r3   r  r  r  r  r  r  r  staticmethodr  r   r   r   r  r   r  r  classmethodr  r  propertyr  r/   r/   r/   r0   rQ    s                          
z	
S

		

    
	

  
     
8
    
	    






6rQ  c                   @   s   e Zd ZdZdZdZdddddedddddfddZdd Zdd Z	d	d
 Z
dd Zdd Zdd Zdd Zd$ddZdd Zd%ddZd&ddZdd Zd'ddZd d! Zd"d# ZdS )(r  Nr/   c                 C   s   || _ t | _t | _tt| _|| _	|| _
|| _|| _|| _|| _|| _|| _|	| _|
| _|pdd| _|| _d| _d| _d | _d | _d | _| || j< d S )Nr/   F)r  r|  _mutexr=   r  _eventr  job_counterr  rY  	_callback_accept_callback_error_callback_timeout_callbackr  r.  r  _on_timeout_set_on_timeout_cancel_callbacks_propagate	_send_ack	_accepted
_cancelledr  r-  _terminated)rH   r   r  r  r  r  r0  rA   r`  rc  rd  r  r  r  r/   r/   r0   rL     s,    


zApplyResult.__init__c                 C   s   dj | jj| j| j|  dS )Nz"<%s: {id} ack:{ack} ready:{ready}>)idZackr   )r   re   r[   r  r  r   rG   r/   r/   r0   rg     s      zApplyResult.__repr__c                 C   s
   | j  S r+   )r  r   rG   r/   r/   r0   r     s    zApplyResult.readyc                 C   s   | j S r+   )r  rG   r/   r/   r0   accepted  s    zApplyResult.acceptedc                 C   s   |   st| jS r+   )r   rr   _successrG   r/   r/   r0   
successful  s    zApplyResult.successfulc                 C   s
   d| _ dS )zOnly works if synack is used.TN)r  rG   r/   r/   r0   _cancel  s    zApplyResult._cancelc                 C   s   | j | jd  d S r+   )rY  popr  rG   r/   r/   r0   discard  s    zApplyResult.discardc                 C   s
   || _ d S r+   )r  rH   rk   r/   r/   r0   r     s    zApplyResult.terminatec                 C   s>   zt |p
d W n& t k
r8   | d dt f Y nX d S Nr   F)r   r   r   r  r/   r/   r0   r    s    zApplyResult._set_terminatedc                 C   s   | j r| j gS g S r+   r  rG   r/   r/   r0   rC    s    zApplyResult.worker_pidsc                 C   s   | j | d S r+   )r  r#  r   r/   r/   r0   r#    s    zApplyResult.waitc                 C   s.   |  | |  st| jr"| jS | jjd S r+   )r#  r   r   r  rQ   	exceptionr   r/   r/   r0   r     s    
zApplyResult.getc              
   O   s^   |rZz||| W nF | j k
r*    Y n0 tk
rX } ztd|dd W 5 d }~X Y nX d S )Nz"Pool callback raised exception: %rr   r   )r  r   r:   )rH   r   r5   r<   rb   r/   r/   r0   safe_apply_callback  s    zApplyResult.safe_apply_callbackFc                 C   s,   | j d k	r(| j| j ||r| jn| jd d S )N)r  rA   )r  r   r.  r  )rH   r  r/   r/   r0   r    s    
 zApplyResult.handle_timeoutc              	   C   s   | j  | jr| |  |\| _| _| j  | jrD| j| j	d  | j
r`| jr`| | j
| j | jd k	r| jr| js| | j| j W 5 Q R X d S r+   )r  r  r  rQ   r  r*  r  rY  r  r  r
  r   r  rH   r   r   r/   r/   r0   r     s*    

 
 zApplyResult._setc                 C   sJ  | j 8 | jrL| jrLd| _|r>| t|| j|W  5 Q R  S W 5 Q R  d S d| _|| _|| _|  rv| j	
| jd  | jr| | | j| j t}| jrd zHz| || W n2 | jk
r   t} Y n tk
r   t}Y nX W 5 | jr|r| ||| j|  W  5 Q R  S X  | jr<|r<| ||| j| W 5 Q R X d S r   )r  r  r  r  r   r  r-  r  r   rY  r  r  r.  r  r   r  Z_propagate_errorsr   )rH   r   rA  r   r   r  r/   r/   r0   r@  $  s>    
   zApplyResult._ack)N)N)N)F)r[   r\   r]   r  r  r  r_  rL   rg   r   r  r  r  r  r   r  rC  r#  r   r   r  r   r@  r/   r/   r/   r0   r    s<        



	

r  c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )r  c                 C   s   t j| |||d d| _|| _d g| | _dg| | _d g| | _d g| | _|| _|dkrvd| _	| j
  || j= n|| t||  | _	d S )Nr  TFr   )r  rL   r  _lengthrQ   r  r  r-  
_chunksize_number_leftr  r*  r  rg  )rH   r   r  lengthr  r  r/   r/   r0   rL   L  s$       

zMapResult.__init__c                 C   s   |\}}|rv|| j || j |d | j < |  jd8  _| jdkr| jrT| | j  | jrj| j| jd  | j	  n>d| _
|| _ | jr| | j  | jr| j| jd  | j	  d S )Nr   r   F)rQ   r#  r$  r
  r  rY  r  r  r  r*  r  r  )rH   r   Zsuccess_resultsuccessr   r/   r/   r0   r   ^  s"    
zMapResult._setc                 G   sj   || j  }t|d | j  | j}t||D ]"}d| j|< || j|< || j|< q*|  rf| j	| j
d  d S Nr   T)r#  rO  r"  r   r  r  r-  r   rY  r  r  )rH   r   rA  r   r5   r   r?   jr/   r/   r0   r@  r  s    


zMapResult._ackc                 C   s
   t | jS r+   )allr  rG   r/   r/   r0   r  |  s    zMapResult.acceptedc                 C   s   dd | j D S )Nc                 S   s   g | ]}|r|qS r/   r/   r  r/   r/   r0   r    s      z)MapResult.worker_pids.<locals>.<listcomp>r  rG   r/   r/   r0   rC    s    zMapResult.worker_pidsN)r[   r\   r]   rL   r   r@  r  rC  r/   r/   r/   r0   r  J  s
   
r  c                   @   sZ   e Zd ZdZefddZdd ZdddZeZdd	 Z	d
d Z
dd Zdd Zdd ZdS )r  Nc                 C   sZ   t t  | _tt| _|| _t | _	d| _
d | _d| _i | _g | _|| _| || j< d S r  )r=   	Conditionr|  rP   r  r	  r  rY  r   _itemsr  r"  _ready	_unsorted_worker_pidsr  )rH   r   r`  r/   r/   r0   rL     s    
zIMapIterator.__init__c                 C   s   | S r+   r/   rG   r/   r/   r0   __iter__  s    zIMapIterator.__iter__c                 C   s   | j  z| j }W nt tk
r   | j| jkr<d| _t| j | z| j }W n. tk
r   | j| jkr|d| _tt	Y nX Y nX W 5 Q R X |\}}|r|S t
|d S r   )rP   r+  popleftr  r  r"  r,  r3  r#  r   r   )rH   rA   rD  r&  rN   r/   r/   r0   r    s&    zIMapIterator.nextc              	   C   s   | j  | j|krn| j| |  jd7  _| j| jkrb| j| j}| j| |  jd7  _q,| j   n
|| j|< | j| jkrd| _| j	| j
= W 5 Q R X d S r'  )rP   r  r+  r  r-  r  rR   r"  r,  rY  r  r!  r/   r/   r0   r     s    

zIMapIterator._setc              	   C   sB   | j 2 || _| j| jkr4d| _| j   | j| j= W 5 Q R X d S r   )rP   r"  r  r,  rR   rY  r  )rH   r%  r/   r/   r0   r    s    
zIMapIterator._set_lengthc                 G   s   | j | d S r+   )r.  r  )rH   r   rA  r   r5   r/   r/   r0   r@    s    zIMapIterator._ackc                 C   s   | j S r+   )r,  rG   r/   r/   r0   r     s    zIMapIterator.readyc                 C   s   | j S r+   )r.  rG   r/   r/   r0   rC    s    zIMapIterator.worker_pids)N)r[   r\   r]   r  r_  rL   r/  r  __next__r   r  r@  r   rC  r/   r/   r/   r0   r    s   
r  c                   @   s   e Zd Zdd ZdS )r  c              	   C   sV   | j F | j| |  jd7  _| j   | j| jkrHd| _| j| j= W 5 Q R X d S r'  )	rP   r+  r  r  rR   r"  r,  rY  r  r!  r/   r/   r0   r     s    
zIMapUnorderedIterator._setN)r[   r\   r]   r   r/   r/   r/   r0   r    s   r  c                   @   s:   e Zd ZddlmZ eZdddZdd Zed	d
 Z	dS )
ThreadPoolr   )rp  Nr/   c                 C   s   t | ||| d S r+   )rQ  rL   )rH   r  rs   rt   r/   r/   r0   rL     s    zThreadPool.__init__c                    s:   t   _t   _ jj _ jj _ fdd}| _d S )Nc                    s.   zd j | dfW S  tk
r(   Y dS X d S r   )r   r   r   rG   r/   r0   r    s    z.ThreadPool._setup_queues.<locals>._poll_result)r   r  ry  r   r   r   r   r  r  r/   rG   r0   rW    s    

zThreadPool._setup_queuesc              	   C   s@   | j 0 | j  | jd gt|  | j   W 5 Q R X d S r+   )	not_emptyqueuerW   extendrP  rT   )r  r  r   r/   r/   r0   r    s    
zThreadPool._help_stuff_finish)NNr/   )
r[   r\   r]   dummyrp  r   rL   rW  r  r  r/   r/   r/   r0   r2    s   
r2  )N)er+  r   r7   r   r   r   r   r=   r   ri  collectionsr   	functoolsr    r   r   r   commonr   r	   r
   r   r   compatr   r   r   r   r   r6  r   
exceptionsr   r   r   r   r   r   r   r   r4  r   r   r   r   r    r   version_infor_   systemZ_winr#   r   r$  r$   r  r,   	SemaphorerK   r   r   r   r   r   r   r   r   r)   r   r   r  r   r_  r   r   countr	  r|  r1   r6   r9   r:   rB   rC   r   r`   rj   rm   rn   r   r   r   r  r6  rQ  r  r  r  r  r2  r/   r/   r/   r0   <module>   s   $	



;  )%K  :     o =R