a
    fezaB                     @   s  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 ddlmZmZmZ ddlmZmZ dd	lmZ ed
Zd dlmZmZmZ d dlm	Z	 d dlmZmZ dd ZdPddZdd Zdd Z dd Z!dd Z"dd Z#ej$%dddgd d! Z&d"d# Z'G d$d% d%e(Z)dQd&d'Z*d(d) Z+d*d+ Z,ej$%d,e-e.dd-d. Z/d/d0 Z0d1d2 Z1d3d4 Z2d5d6 Z3d7d8 Z4d9d: Z5d;d< Z6d=d> Z7d?d@ Z8dAdB Z9dCdD Z:ej$%dEdFdGgej$j;ej<dHkoej<dIkdJdKdLdM Z=dNdO Z>dS )R    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_backend)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributed)ClientLocalCluster
get_client)time)clusterincc                  O   s   d S N )argskwargsr   r   4lib/python3.9/site-packages/joblib/test/test_dask.pynoop   s    r   皙?c                 C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)Z	conditionZdurationr   r   r   slow_raise_value_error   s    r   c                    sF   | dd }i }| D ]&\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>       zcount_events.<locals>.<lambda>c                    s   g | ]}|d   kr|qS    r   ).0Zevent
event_namer   r   
<listcomp>!   s   z count_events.<locals>.<listcomp>)runitemslenlist)r'   clientZworker_eventsZevent_countswZeventsr   r&   r   count_events   s
     r/   c                 C   s:  t  \}\}}t|d | d}td\}}t dd tdD }|dd tdD ksdJ tt( t d	d tdD  W d    n1 s0    Y  t d
d tdD }|dd tdD ksJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s,0    Y  d S )Naddressloopdaskc                 s   s   | ]}t t|V  qd S r   r   r   r%   ir   r   r   	<genexpr>*   r"   ztest_simple.<locals>.<genexpr>
   c                 S   s   g | ]}t |qS r   r   r5   r   r   r   r(   +   r"   ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t t|d kV  qdS )   N)r   r   r5   r   r   r   r7   .   s   c                 s   s   | ]}t t|V  qd S r   r4   r5   r   r   r   r7   1   r"   c                 S   s   g | ]}t |qS r   r9   r5   r   r   r   r(   2   r"   )r   r   r   r
   rangepytestraisesr   r2   sabr-   ba_seqr   r   r   test_simple&   s    
&rE   c           	      C   s  t jtju sJ t \}\}}t|d | d}td\}}t h}|j}t|t s\J |j	|u sjJ |j
dksxJ |dd ttdD  |j
dksJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )	Nr0   r1   r3   r$   c                 s   s   | ]}t d d  V  qdS )c                   S   s   d S r   r   r   r   r   r   r!   G   r"   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r%   rC   r   r   r   r7   F   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @r8   )r   Zcompute_batch_sizer   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizer;   int)	r2   r?   r@   rA   r-   rB   rC   rK   Zbackendr   r   r   #test_dask_backend_uses_autobatching5   s    
rM   c                   C   s   t  S r   r   r   r   r   r   random2M   s    rN   c           	   
   C   s   t  \}\}}t|d | dd}td<\}}t dd tdD \}}||ksXJ W d    n1 sl0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr0   r1   r3   c                 s   s   | ]}t t V  qd S r   )r   rN   r5   r   r   r   r7   U   r"   z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   r;   )	r2   r?   r@   rA   r-   rB   rC   xyr   r   r    test_dont_assume_function_purityQ   s
    rR   mixedTFc              
   C   s&  ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRJ t \}\}}t|d	 | d
}td$\}	}
tddd|}
W d    n1 s0    Y  dd }|dd}|	|}t
dd |D sJ W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r4   r5   r   r   r   r(   ]   r"   z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt t|n
t t|qS )r	   )r   absr   r5   r   r   r   r(   `   s   Zmixed_batch_of_inc_4_callsr0   r1   r3   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r,   Ztransition_log)Zdask_schedulerr   r   r   fl   s    ztest_dask_funcname.<locals>.f42c                 s   s   | ]}d |d v V  qdS )Zbatch_of_incr   Nr   )r%   tupr   r   r   r7   p   r"   z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskrT   r;   reprr   r   r   r
   replaceZrun_on_schedulerrW   )r2   rS   rT   tasksZ
batch_reprr?   r@   rA   r-   rB   rC   rX   r    r   r   r   test_dask_funcnameY   s"    .
r_   c                    s  dd t dD }tdtd d fdd	tdd	d
}t|}z&td,\}}t fdd|D }W d    n1 s0    Y  |dd t dD ksJ t	d|}t
| dksJ tdd |D sJ td.\}}t  fdd|D }W d    n1 s"0    Y  t	d|}t
| dksLJ tdd |D sdJ W |  |  n|  |  0 d S )Nc                 S   s   g | ]}g qS r   r   rG   r   r   r   r(   ~   r"   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d urj |  | t j | S r   )ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr$   r	   )	n_workersthreads_per_workerr3   c                 3   s   | ]}t  |V  qd S r   rF   r%   rd   )rh   r   r   r7      s   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g qS r   r   rG   r   r   r   r(      r"   receive-from-scatterr   c                 S   s   g | ]}t |d kqS r#   r+   r%   rr   r   r   r(      r"   c                 3   s   | ]}t | d V  qdS ))re   NrF   rk   )rf   rh   r   r   r7      s   c                 S   s   g | ]}t |d kqS r#   rm   rn   r   r   r   r(      r"   )N)r;   r<   importorskipZarangerL   r   r   r   r
   r/   sumvaluesrW   close)r2   Zlistsr   r-   rB   rC   rescountsr   )rf   rh   rg   r   'test_no_undesired_distributed_cache_hits   s4    
&
(

rv   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )rP   count)selfrP   r   r   r   __init__   s    zCountSerialized.__init__c                 C   s   | j t|d| S )NrP   )rP   getattr)ry   otherr   r   r   __add__   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr$   )rx   rw   rP   )ry   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__rz   r}   __radd__r~   r   r   r   r   rw      s   rw   c                 C   s   | | | | | S r   r   )r@   rA   cder   r   r   add5   s    r   c                 C   s  t d}t d}t d}t @\}\}}t|d | d}td||gdx\}}	tt}
|
|||dd	d
|
|||d	dd
|
||||d	d
|
|||||d
g}dd |D }t |}W d    n1 s0    Y  tt	@ td| dd W d    n1 s0    Y  W d    n1 s(0    Y  W d    n1 sH0    Y  W d    n1 sh0    Y  ||ksJ |j
dksJ |j
dksJ |j
dv sJ d S )Nr$   r	   r:   r0   r1   r3   )scatterrU      )r   r   c                 S   s    g | ]\}}}||i |qS r   r   )r%   funcr   r   r   r   r   r(      s   z'test_manual_scatter.<locals>.<listcomp>)r2   r   )rU      )rw   r   r   r   r   r   r
   r<   r=   	TypeErrorrx   )r2   rP   rQ   zr?   r@   rA   r-   rB   rC   rX   r^   Zexpectedresultsr   r   r   test_manual_scatter   s.    (r   c              
      s  t d}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d~}td,\}}	t dd	 t	|D  W d    n1 s0    Y  t
d
|}
|
|d  |
|d   dksJ W d    n1 s0    Y  W d    n1 s
0    Y  t \}\}}t|d | d}td0\}}	t  fdd	tdD  W d    n1 sv0    Y  t
d
|}
|
|d  dksJ |
|d  dksJ W d    n1 s0    Y  W d    n1 s0    Y  d S )Nra   rH   )Zdtyper:   r0   r1   r3   c                 s   s&   | ]\}}t t||||d V  qdS ))optNr   r   )r%   r6   re   r   r   r   r7      s   z$test_auto_scatter.<locals>.<genexpr>rl   r	   c                 3   s$   | ]}t t d d |V  qd S )Nr:   r   r5   Zdata1r   r   r7      r"   r   r   )r<   rp   onesrL   Zuint8r   r   r   r
   	enumerater/   r;   )loop_in_threadrg   Zdata2Zdata_to_processr?   r@   rA   r-   rB   rC   ru   r   r   r   test_auto_scatter   s&    

&
^<
r   retry_noc              
      s   t dd d}fdd fddt \}\}}t|d | d`}td	8 d
t fddt|D }W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nra   r8   c                    s
     | S r   )rq   )rP   r6   j)rg   r   r   my_sum
  s    z#test_nested_scatter.<locals>.my_sumc                    sV   t  }td0 t  fddtD }W d    n1 sD0    Y  t|S )Nr3   c                 3   s&   | ]}t  |d  |V  qd S r   rF   )r%   r   )arrayr6   r   r   r   r7     s   zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   r;   rq   )r   r6   r-   r   )NUM_INNER_TASKSr   )r   r6   r   outer_function_joblib  s    
&z2test_nested_scatter.<locals>.outer_function_joblibr0   r1   r3   i'  c                 3   s$   | ]}t  |d  |V  qd S r   rF   r5   )my_arrayr   r   r   r7     s   z&test_nested_scatter.<locals>.<genexpr>)r<   rp   r   r   r   r   r
   r;   )r2   r   ZNUM_OUTER_TASKSr?   r@   rA   rC   r   )r   r   r   rg   r   r   test_nested_scatter  s    
	

r   c           	   
      sv  dd  t  P\}\}}t|d | dz}tdR\}}tdd fdd	td
D }|D ]}tt|dks^J q^W d    n1 s0    Y  W d    n1 s0    Y  t|d | d|}tdR\}}tdd fdd	td
D }|D ]}tt|dksJ qW d    n1 s(0    Y  W d    n1 sH0    Y  W d    n1 sh0    Y  d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s   s   | ]}t tj V  qd S r   r   osgetpidrG   r   r   r   r7   "  r"   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tj V  qd S r   r   rG   r   r   r   r7   #  r"   )setr
   r;   )Zpidsr   r   r   get_nested_pids!  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr0   r1   r3   r	   r   c                 3   s   | ]}t   V  qd S r   rF   rG   r   r   r   r7   )  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r8   c                 3   s   | ]}t   V  qd S r   rF   rG   r   r   r   r7   3  s   )r   r   r   r
   r;   r+   r   )	r   r?   r@   rA   r-   rB   rC   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager   s     Rr   c                    s(  dd   fddt  \}\}}t|d | d}td\}}t H} |dks\J |jd	ksjJ |fd
dtdD }W d    n1 s0    Y  |D ] \}	}
|	dksJ |
d	ksJ qW d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nc                 S   s
   | j jjS r   )rI   	__class__r   pr   r   r   _backend_type?  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t  }  | | jfW  d    S 1 s*0    Y  d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsB  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr0   r1   r3   r   c                 3   s   | ]}t   V  qd S r   rF   rG   )r   r   r   r7   L  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r   r;   )r2   r?   r@   rA   r-   rB   rC   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs;  s    &r   c              	   C   sl   t t8}td W d    n1 s*0    Y  W d    n1 sH0    Y  dt|j v shJ d S )Nr3   zcreate a dask client)r<   r=   r   r   strvaluelower)r2   infor   r   r   test_errorsU  s    
<r   c              
   C   s*  t  \}\}}t|d | d}tdJ\}}tdddd tdD }t|d	 d	 d	 tshJ W d    n1 s|0    Y  tdJ\}}tddd
d tdD }t|d	 d	 d	 tsJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr0   r1   r3   r	   r   c                 s   s   | ]}t td dV  qd S )NZnested_requirer   outerrG   r   r   r   r7   b  s   z.test_correct_nested_backend.<locals>.<genexpr>r$   r   c                 s   s   | ]}t td dV  qdS )Z	sharedmemr   Nr   rG   r   r   r   r7   h  s   )r   r   r   r
   r;   rJ   r   r   )r2   r?   r@   rA   r-   rB   rC   resultr   r   r   test_correct_nested_backend]  s    8r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s   | ]}t t V  qd S r   )r   middlerG   r   r   r   r7   o  s   zouter.<locals>.<genexpr>r$   r
   r;   r   r   r   r   r   n  s    r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   s   | ]}t t V  qd S r   )r   innerrG   r   r   r   r7   u  s   zmiddle.<locals>.<genexpr>r$   r   )r   r   r   r   r   t  s    r   c                   C   s   t  jS r   )r
   rI   r   r   r   r   r   z  s    r   c              	   C   sv   t | dddT td, tdddd td	D  W d    n1 sJ0    Y  W d    n1 sh0    Y  d S )
NFT)r2   	processesZset_as_defaultr3   rU   r   c                 s   s   | ]}t t|V  qd S r   )r   idr5   r   r   r   r7     r"   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   r;   r1   r   r   r   test_secede_with_no_processes~  s    
r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r0   )rC   r   r   r   r   _worker_address  s    r   c              
   C   s*  t  \}\}}t|d | d}td|d dB\}}t dd tdD }||d gd kshJ W d    n1 s|0    Y  td|d dB\}}t dd tdD }||d gd ksJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )	Nr0   r1   r3   )workersc                 s   s   | ]}t t|V  qd S r   r   r   r5   r   r   r   r7     s   z-test_dask_backend_keywords.<locals>.<genexpr>r8   c                 s   s   | ]}t t|V  qd S r   r   r5   r   r   r   r7     s   rO   r>   r   r   r   test_dask_backend_keywords  s    
4
r   c              	   C   s   t d| d}td( t dd tdD  W d    n1 sD0    Y  t }|jjjrztd t |d k sTJ qT|j	rJ W d    n1 s0    Y  d S )	NF)r   r2   r3   c                 s   s   | ]}t t|V  qd S r   r4   r5   r   r   r   r7     r"   ztest_cleanup.<locals>.<genexpr>r8   g{Gz?r   )
r   r   r
   r;   r   r   Z	schedulerr^   r   Zfutures)r2   r-   startr   r   r   test_cleanup  s    
6
r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                 C   s   t dddd}t|}| dkr.|jddd n| dkr@|d zTtd( t d	d
 tdD  W d    n1 sx0    Y  W |  |  n|  |  0 d S )Nr   Fr	   ri   r   rj   r   )ZminimumZmaximumr   r3   c                 s   s   | ]}t t|V  qd S r   r4   r5   r   r   r   r7     r"   z(test_wait_for_workers.<locals>.<genexpr>r8   )r   r   ZadaptZscaler   r
   r;   rs   )r   r   r-   r   r   r   test_wait_for_workers  s    

8
r   c               	   C   s0  t dddd} t| }z tdddZ d}tjt|d	( t d
d tdD  W d    n1 sh0    Y  W d    n1 s0    Y  tdddZ d}tjt|d	( t dd tdD  W d    n1 s0    Y  W d    n1 s0    Y  W |	  | 	  n|	  | 	  0 d S )Nr   Fr	   r   r3   g?)Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   s   | ]}t t|V  qd S r   r4   r5   r   r   r   r7     r"   z0test_wait_for_workers_timeout.<locals>.<genexpr>r8   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t t|V  qd S r   r4   r5   r   r   r   r7     r"   )
r   r   r   r<   r=   TimeoutErrorr
   r;   RuntimeErrorrs   )r   r-   msgr   r   r   test_wait_for_workers_timeout  s    TV
r   )r   )r   r   )?Z
__future__r   r   r   r   r<   r   Zuuidr   r   r    r
   r   r   rK   r   r   Z_daskr   rp   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r   r/   rE   rM   rN   rR   ZmarkZparametrizer_   rv   objectrw   r   r   r   r,   r;   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   <module>   s\   

	
9
%
