a
    feza
                  	   @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlm Z m!Z! ddlm"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z( dddZ)dd Z*dd Z+dd Z,dd Z-e$dddgdd Z.d d! Z/d"d# Z0d$d% Z1d&d' Z2d(d) Z3d*d+ Z4d,d- Z5d.d/ Z6g fd0d1Z7d2d3 Z8e e$d4dd5gd6d7 Z9e d8d9 Z:d:d; Z;d<d= Z<d>d? Z=d@dA Z>e$dBdCgdDd5fg dEdfgdFdG Z?dHdI Z@dJdK ZAdLdM ZBdNdO ZCdPdQ ZDdRdS ZEdTdU ZFdVdW ZGdXdY ZHdZd[ ZId\d] ZJd^d_ ZKd`da ZLdbdc ZMdddedfdgdhZNeOePePdidjdkZQdldm ZRdndo ZSddpdqZTdrds ZUdtdu ZVdvdw ZWdxdy ZXdzd{ ZYdd|d}ZZe"d~d Z[e"dd Z\dd Z]dd Z^G dd deZ_G dd deZ`e$ddea eb gdd Zcdd Zddd Zedd Zfdd Zgdd Zhdd Zidd Zjdd Zkdd ZldddZmejn$ddddd5ddddidgdd ZodS )z
Test the memory module.
    N)Memory)MemorizedFuncNotMemorizedFunc)MemorizedResultNotMemorizedResult)_FUNCTION_HASHES)register_store_backend_STORE_BACKENDS)_build_func_identifier_store_backend_factory)JobLibCollisionWarning)Paralleldelayed)StoreBackendBaseFileSystemStoreBackend)
with_numpynp)with_multiprocessing)parametrizeraiseswarns)hash   c                 C   s   | d | S )z3 A module-level function for testing purposes.
        xyr   r   6lib/python3.9/site-packages/joblib/test/test_memory.pyf)   s    r   c                 C   sZ   t |dd}|| } tdD ]6}tdD ](}| ||ks>J t||d ks*J q*qdS )z Given a function and an accumulator (a list that grows every
        time the function is called), check that the function can be
        decorated by memory to be a lazy identity.
    r   locationverbose   r   r   N)r   cacherangelen)funcaccumulatorr!   memoryi_r   r   r   check_identity_lazy1   s    
r,   c                 C   sT   | j  \}tj|jd}t|d}|d W d    n1 sF0    Y  d S )N
output.pklwZgarbage)store_backend	get_itemsospathjoinopenwrite)r)   Zsingle_cache_itemZoutput_filenamer   r   r   r   corrupt_single_cache_item@   s    r6   c                    s"   g   fdd}| | d|  S )Nc                    s     |  d S Nappend)itemZrecordedr   r   append_to_recordN   s    z6monkeypatch_cached_func_warn.<locals>.append_to_recordwarn)setattr)r'   Zmonkeypatch_fixturer<   r   r;   r   monkeypatch_cached_func_warnG   s    r?   c                    s   t    fdd}t| | j dD ]}dD ]T}t| jd||d}tj| jdd ||}|d	 |jd
d t }|d	}q,t |d	 ksJ |	|d	|ksJ t |d	 ks$J q$d|_
t| jdd}||d	 dS )z, Simple test of memory lazy evaluation.
    c                    s     d | S Nr   r8   lr(   r   r   r   ^   s    
z"test_memory_integration.<locals>.f)FT)rN
   )r!   r"   	mmap_modecompressT)ignore_errorsr   F)r=   __main__r   r    N)listr,   strpathr   shutilrmtreer$   clearr&   eval
__module__)tmpdirr   rG   rF   r)   gcurrent_accumulatoroutr   rC   r   test_memory_integrationV   s(    

rU   call_before_reducingTFc           
         s  dD ]}d}d |}ttt||dd t d }d|_|ddksPJ |jj|ks`J t	| j
d	d
}|| tt| d dksJ t| d d	 }d|v sJ | d | }|d	krt|dgksJ t|d g ksJ |r8 d tt|d dksJ tdd fdddD }	n@tdd fdddD }	tt|d dkspJ  d tt|d dksJ qtt|d dksJ  d tt|d dksJ qd S )N)r   r   z0
        def f(x):
            return x
        z<ipython-input-{}-000000000000>exec)filenamemoder   rI   r   r   r    joblibzipython-inputr#   r   Zn_jobsc                 3   s   | ]}t  |V  qd S r7   r   .0r*   Zcached_fr   r   	<genexpr>       zHtest_parallel_call_cached_function_defined_in_jupyter.<locals>.<genexpr>)r   r   c                 3   s   | ]}t  |V  qd S r7   r\   r]   r_   r   r   r`      ra      )formatrW   compiletextwrapdedentlocalsrP   __code__co_filenamer   rK   r$   r&   r1   listdirr   )
rQ   rV   Z
session_noZipython_cell_sourceZipython_cell_idZ	aliased_fr)   Zf_cache_relative_directoryZf_cache_directoryresr   r_   r   5test_parallel_call_cached_function_defined_in_jupyter   sD    





	rl   c                     s^   t    fdd} tddd}|| }tdD ](}t }|d t |d ks0J q0dS )z, Test memory with location=None: no memoize c                    s     d | S r@   r8   rA   rC   r   r   ff   s    
ztest_no_memory.<locals>.ffNr   r    rb   r   )rJ   r   r$   r%   r&   )rm   r)   Zggr+   rS   r   rC   r   test_no_memory   s    
rn   c                    sR   t   d
 fdd	}t| | j t| jdd}||}|ddd	dksNJ dS )z4 Test memory with a function with keyword arguments.Nr   c                    s     d | S r@   r8   rB   mrC   r   r   rR      s    
ztest_memory_kwarg.<locals>.gr   r       r   ro   )Nr   )rJ   r,   rK   r   r$   )rQ   rR   r)   r   rC   r   test_memory_kwarg   s    
rr   c                    s0   t    fddfdd}t| | j dS )z+ Test memory with a function with a lambda.c                    s     d | S z4 A helper function to define l as a lambda.
        r   r8   r   rC   r   r   helper  s    
z"test_memory_lambda.<locals>.helperc                    s    | S r7   r   rt   )ru   r   r   <lambda>  ra   z$test_memory_lambda.<locals>.<lambda>N)rJ   r,   rK   )rQ   rB   r   )r(   ru   r   test_memory_lambda  s    rw   c                 C   s   t | jdd}|jdd }|}|jdd }|}tt }|d |d W d   n1 s`0    Y  t|dkszJ dt|d jv sJ dS )	z> Check that name collisions with functions will raise warningsr   r    c                 S   s   | S )z0 A first function called name_collision
        r   rt   r   r   r   name_collision  s    z2test_memory_name_collision.<locals>.name_collisionc                 S   s   | S )z1 A second function called name_collision
        r   rt   r   r   r   rx     s    r   NZ	collision)r   rK   r$   r   r   r&   strmessage)rQ   r)   rx   abwarninfor   r   r   test_memory_name_collision  s    


&r~   c                 C   s   t | jdd}dd }||}dd }||}tt@}|ddksLJ |ddks\J |ddkslJ W d    n1 s0    Y  t|dksJ d S )	Nr   r    c                 S   s   | S r7   r   rt   r   r   r   rv   2  ra   z7test_memory_warning_lambda_collisions.<locals>.<lambda>c                 S   s   | d S r@   r   rt   r   r   r   rv   4  ra   r   r   rb   )r   rK   r$   r   r   r&   )rQ   r)   r{   r|   r}   r   r   r   %test_memory_warning_lambda_collisions/  s    


.r   c                 C   s   t | jdd}td}||}td}||}tt(}|d |d |d W d    n1 sh0    Y  t|dksJ dt|d j	 v sJ d S )Nr   r    lambda x: xzlambda x: x+1r   r   zcannot detect)
r   rK   rO   r$   r   r   r&   ry   rz   lower)rQ   r)   Za1b1r}   r   r   r   'test_memory_warning_collision_detectionA  s    


&r   c                    s8   t    fdd}ddl}||d}t| | j dS )z$ Test memory with functools.partial.c                    s     d |S rs   r8   r   rC   r   r   r'   W  s    
z!test_memory_partial.<locals>.funcr   Nr   )rJ   	functoolspartialr,   rK   )rQ   r'   r   functionr   rC   r   test_memory_partialS  s
    r   c                 C   s4   t | jdd}td}||}|ddks0J dS )zF Smoke test memory with a function with a function defined in an eval.r   r    r   r   N)r   rK   rO   r$   )rQ   r)   rp   Zmmr   r   r   test_memory_evalc  s    
r   c                 C   s   t | }| d |S )zx A function with a side effect in its arguments.

        Return the length of its argument and append one element.
    N)r&   r9   )r   Zlen_xr   r   r   count_and_appendm  s    
r   c                 C   s8   t | jdd}|t}| dks&J | dks4J dS )zx Check that if a function has a side effect in its arguments, it
        should use the hash of changing arguments.
    r   r    r   N)r   rK   r$   r   )rQ   r)   r'   r   r   r   test_argument_changew  s    
r   rF   rD   c           	         s   t   d	 fdd	}t| j|dd}||}tjd}tdD ]F}|d}tdD ].}t	|||ksrJ t
 |d ksXJ qXqBdS )
z/ Test memory with a function with numpy arrays.Nc                    s     d | S r@   r8   rA   rC   r   r   n  s    
ztest_memory_numpy.<locals>.nr   r!   rF   r"   r#   )rE   rE   r   )N)rJ   r   rK   r$   r   ZrandomZRandomStater%   Zrandom_sampleallr&   )	rQ   rF   r   r)   Zcached_nZrndr*   r{   r+   r   rC   r   test_memory_numpy  s    

r   c           
      C   s   t | jddd}| dd }td}||}||}t|tjsJJ |jdksXJ t|tjshJ |jdksvJ ~~t	  t
| t||}||}t|dksJ d}	|	|d v sJ t|tjsJ |jdksJ d	S )
z8Check that mmap_mode is respected even at the first callrD   r   r   c                 S   s   | d S Nr   r   r{   r   r   r   twice  s    z0test_memory_numpy_check_mmap_mode.<locals>.twicer#   r   Exception while loading resultsN)r   rK   r$   r   Zones
isinstanceZmemmaprY   gcZcollectr6   r?   r&   )
rQ   monkeypatchr)   r   r{   r|   crecorded_warningsdexception_msgr   r   r   !test_memory_numpy_check_mmap_mode  s*    


r   c              	      sz   t | jdd}G dd dt |jd
 fdd	}|  tdD ]4}t  |d W d	   q@1 sj0    Y  q@d	S )z1 Smoketest the exception handling of Memory.
    r   r    c                   @   s   e Zd ZdS )z*test_memory_exception.<locals>.MyExceptionN__name__rP   __qualname__r   r   r   r   MyException  s   r   c                    s   | r d S r7   r   )excr   r   r   h  s    z test_memory_exception.<locals>.hr#   r   N)r   )r   rK   	Exceptionr$   r%   r   )rQ   r)   r   r+   r   r   r   test_memory_exception  s    
r   c                    s   t | jdd}t  |jdgdd fdd	}|jdgks@J |ddd t dks\J |ddd t dksxJ |dd	d t dksJ d
S )z# Test the ignore feature of memory r   r    r   ignorer   c                    s     d d S r@   r8   r   rC   r   r   z  s    ztest_memory_ignore.<locals>.zr   r   N)r   r   rK   rJ   r$   r   r&   )rQ   r)   r   r   rC   r   test_memory_ignore  s    r   c                    s   t | jdd}t  dd }|jdgd|d fdd		}|jdgksLJ |ddd
 t dkshJ |ddd
 t dksJ |ddd
 t dksJ dS )z; Test the ignore feature of memory on a decorated function r   r    c                    s   t   fdd}|S )Nc                     s    | i |S r7   r   )argskwargsr   r   r   wrapped  s    z?test_memory_ignore_decorated.<locals>.decorate.<locals>.wrapped)r   wraps)r   r   r   r   r   decorate  s    z.test_memory_ignore_decorated.<locals>.decorater   r   r   c                    s     d d S r@   r8   r   rC   r   r   r     s    z'test_memory_ignore_decorated.<locals>.zr   r   N)r   r   )rQ   r)   r   r   r   rC   r   test_memory_ignore_decorated  s    r   c                 C   sT   t | jdd}|jdd }|ddks,J |dddks>J |dddksPJ d	S )
z_Non-regression test against 0.12.0 changes.

    https://github.com/joblib/joblib/pull/751
    r   r    c                 S   s   | d S r@   r   r   r   r   r   plus_one  s    z,test_memory_args_as_kwargs.<locals>.plus_oner   r   r   r#   N)r   rK   r$   )rQ   r)   r   r   r   r   test_memory_args_as_kwargs  s    
r   zignore, verbose, mmap_moder   d   rE   c                 C   sT   t | jdd}|j|||ddd }|j|ks4J |j|ksBJ |j|ksPJ dS )z7Check cache may be called with kwargs before decoratingr   r    )r   r"   rF   c                 S   s   d S r7   r   rt   r   r   r   r      s    z"test_partial_decoration.<locals>.zN)r   rK   r$   r   Z_verboserF   )rQ   r   r"   rF   r)   r   r   r   r   test_partial_decoration  s    
r   c           
      C   s  t | jdd}td}|d | jdg|R  j}|t}tt}t	j
|jj|}||ksfJ t	j
|svJ |jt	j
|jjksJ tt"}|j|jjksJ W d    n1 s0    Y  t|dksJ dt|d jv sJ t  | rJ t	j
t	j
|d	s"J | s0J |d\}}t	j
|jj||}|d}	t	j
|slJ t	t	j
|d
 |	|dksJ d S )Nr   r    .r   rZ   r   ,The 'cachedir' attribute has been deprecatedzfunc_code.pyr-   )r   rK   r   splitr9   r3   r$   r   r
   r1   r2   r/   r!   existsdirnamer   DeprecationWarningcachedirr&   ry   rz   r   rN   Z_check_previous_func_code_get_output_identifiersremove)
rQ   r)   r2   rR   func_idr!   r.   args_id
output_dirr{   r   r   r   test_func_dir)  s0    



0r   c           
      C   s   t | jdd}|t}|d}tt|}|d\}}tj	
|jj||}tj	|sbJ ||j||gkszJ tt|}|jj|jjksJ t d dd}tt| |t}tt|}	|	d d S )Nr   r    r   )r   rK   r$   r   pickleloadsdumpsr   r1   r2   r3   r/   r!   r   Z	load_item)
rQ   r)   rR   outputr   r   r   r   Zmemory2Zgpr   r   r   test_persistenceM  s    

r   c                 C   s   t t| jt| jddtfD ]\}|d}|r6J t|tsDJ |ddksTJ |d}|sfJ t|tstJ |  q d S )Nr   r    r      )	r   r   rK   r   r$   Zcheck_call_in_cacher   boolrN   )rQ   r'   resultr   r   r   test_check_call_in_cached  s    


r   c              	   C   s   t tt| jttt| jddttd dtfttttfD ]\}}|ddksZJ |	d}t
||srJ | dksJ |  tt |  W d    n1 s0    Y  |  qBd S )Nr   r    r!   r   r   )zipr   r   rK   r   r   r$   r   r   call_and_shelver   getrN   r   KeyError)rQ   r'   ZResultr   r   r   r   test_call_and_shelveq  s*    

&r   c                 C   s   t | jddt}|d}t|ts,J tt }|j	|j
ksFJ W d    n1 sZ0    Y  t|dkstJ dt|d jv sJ d S )Nr   r    r   r   z1The 'argument_hash' attribute has been deprecatedr   )r   rK   r$   r   r   r   r   r   r   argument_hashr   r&   ry   rz   )rQ   r'   r   r.   r   r   r   "test_call_and_shelve_argument_hash  s    

.r   c           
      C   s  |  d}|d t|jj}td | dks<J |t|jjkrXt	
d t| jdd}|t}|d\}}tj |jj||d}|ddksJ t|j}td	 |d}	t|	tsJ t|j|ksJ td	 |	 dksJ t|j|ksJ d
S )z6Check call_and_shelve only load stored data if needed.Ztest_accessg      ?z>filesystem does not support fine-grained access time attributer   r    r   r-   r   r   N)r3   r5   r1   statrK   st_atimetimesleepreadpytestskipr   r$   r   r   r2   r/   r!   r   r   r   r   )
rQ   Ztest_access_time_fileZtest_access_timer)   r'   r   r   result_pathZfirst_access_timer   r   r   r   .test_call_and_shelve_lazily_load_stored_result  s,    







r   c              	   C   s   t t| jttfD ]}| dj}|d}t|d}t|| W d    n1 sZ0    Y  t|d}t	|}W d    n1 s0    Y  |
 |
 ksJ t| qd S )Nzpickling_test.datr   wbrb)r   r   rK   r   r3   r   r4   r   dumploadr   r1   r   )rQ   r'   rX   r   fpresult2r   r   r   test_memorized_pickling  s    
*(r   c                 C   s
  t t| j}|d}t t| j}|d}| | ks@J t|t|ksTJ tt}t| t|d t t| jdt d}|d}|  t t| jdd}|d}|  t t| jdt d}|d}|  t t| jdd}|d}|  d S )Nr      )r"   	timestamp)r"   r   )r   r   rK   r   r   reprr   r   )rQ   r'   r   Zfunc2r   r   r   r   test_memorized_repr  s*    





r   c                 C   s  | dj}tj|d}d}t|d}|| W d    n1 sH0    Y  || dd l}t	|jdd}|
|j}	|	d |	d |	d t|d}|d	|  W d    n1 s0    Y  |	d |	d t| t | d
}t|d}|| W d    n1 s"0    Y  |	d |	d tjd tjd dd l}|
|j}	|	d |	d |  \}
}|
dksJ d S )NZ
tmp_importztmp_joblib_.pyz$def f(x):
    print(x)
    return x
r.   r   r    r   r   z

z-def f(x):
    print("x=%s" % x)
    return x
z
Reloading
tmp_joblib_z1
2
Reloading
x=1
)mkdirrK   r1   r2   r3   r4   r5   Zsyspath_prependr   r   r$   r   rL   rM   sysstdoutmodulespop
readouterr)ZcapsysrQ   r   Zdir_namerX   ZcontentZmodule_filetmpr)   r   rT   errr   r   r   test_memory_file_modification  s>    (
,

*r   c                 C   s   d S r7   r   r{   r|   r   r   r   _function_to_cache  s    r   c                 C   s   | | S r7   r   r   r   r   r   _sum!  s    r   c                 C   s   | | S r7   r   r   r   r   r   _product%  s    r   c                 C   s   t jt_t| jdd}|t}|dddks2J |dddksDJ tt< tjt_|dddkshJ |dddkszJ W d    n1 s0    Y  d S )Nr   r    r   r   r#   )	r   rh   r   r   rK   r$   r   r   r   )rQ   r)   r   r   r   r   *test_memory_in_memory_function_code_change)  s    

r   c                  C   s   t d d} |   d S )Nr   )r   rN   )r)   r   r   r   $test_clear_memory_with_none_location9  s    
r   kw1kw2r   r   c                C   s   | |||fS r7   r   )r{   r|   r   r   r   r   r   func_with_kwonly_args>  s    r   )r{   r|   returnc                 C   s   | | S r7   r   r   r   r   r   func_with_signatureB  s    r   c                 C   s  t | jdd}|t}|dddddks.J tt }|ddddd	 W d    n1 s\0    Y  |d
 |ddddd tt }|ddddd	 W d    n1 s0    Y  |d
 |jtdgd}|ddddddksJ |ddddddksJ d S )Nr   r    r   r   r#   )r   )r   r   r#   r   rb   )r   z?Keyword-only parameter 'kw1' was passed as positional parameterr   r   r   )r   r   r#   rb   Zignored)r   rK   r$   r   r   
ValueErrormatch)rQ   r)   func_cachedexcinfor   r   r   !test_memory_func_with_kwonly_argsF  s    

.

.
r  c                 C   s.   t | jdd}|t}|dddks*J d S )Nr   r    r   g       @g      @)r   rK   r$   r   )rQ   r)   r   r   r   r   test_memory_func_with_signatureb  s    
r  c                    st   t | jdd}| dd tt|}|D ]}| q.t fdd|D } fdd|D }||fS )Nr   r    c                 S   s   dS )N  aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar   argr   r   r   get_1000_bytesl  s    z(_setup_toy_cache.<locals>.get_1000_bytesc                    s   g | ]}  |d  qS )r   )r   )r^   r  )r  r   r   
<listcomp>u  s   z$_setup_toy_cache.<locals>.<listcomp>c                    s    g | ]}t jjj |qS r   )r1   r2   r3   r/   r!   )r^   r   r   r  r   r   r  x  s   )r   rK   r$   rJ   r%   r
   )rQ   Z
num_inputsr)   Zinputsr  Zhash_dirnamesZfull_hashdirsr   r	  r   _setup_toy_cachei  s    


r
  c                    s   t | \}}}|j }dd |D }t|t|ks:J dd   fdd|D }dd |D }||ksnJ dd |D }dd |D }	d	d |D }
|
|	ksJ d S )
Nc                 S   s   g | ]
}|j qS r   )r2   r^   Zcir   r   r   r    ra   z#test__get_items.<locals>.<listcomp>c                    s*    fddt  D }tdd |D S )Nc                    s   g | ]}t j |qS r   r1   r2   r3   r^   fn	directoryr   r   r    s   z;test__get_items.<locals>.get_files_size.<locals>.<listcomp>c                 s   s   | ]}t j|V  qd S r7   )r1   r2   getsize)r^   r   r   r   r   r`     ra   z:test__get_items.<locals>.get_files_size.<locals>.<genexpr>)r1   rj   sum)r  Z
full_pathsr   r  r   get_files_size  s    
z'test__get_items.<locals>.get_files_sizec                    s   g | ]} |qS r   r   r^   Zhash_dirr  r   r   r    s   c                 S   s   g | ]
}|j qS r   )sizer  r   r   r   r    ra   c                 S   s   g | ]}t j|d qS )r-   r  r  r   r   r   r    s   c                 S   s    g | ]}t j tj|qS r   )datetimeZfromtimestampr1   r2   getatimer  r   r   r   r    s   c                 S   s   g | ]
}|j qS r   Zlast_accessr  r   r   r   r    ra   )r
  r/   r0   set)rQ   r)   Zexpected_hash_dirsr+   itemsZ	hash_dirsZexpected_hash_cache_sizesZhash_cache_sizesZoutput_filenamesZexpected_last_accessesZlast_accessesr   r  r   test__get_items~  s$    

r  c                 C   s   t | \}}}|j }|jd}t|}tt|t|sDJ t||d ksXJ |jd}t|t|ksxJ |jd}|g ksJ d}	|j|	}
t|
sJ t||jd}t||}t	dd |D t
d	d |D ksJ d S )
NZ2Kr   i   1M  ip  c                 s   s   | ]}|j V  qd S r7   r  r  r   r   r   r`     ra   z,test__get_items_to_delete.<locals>.<genexpr>c                 s   s   | ]}|j V  qd S r7   r  r  r   r   r   r`     ra   )r
  r/   r0   Z_get_items_to_deleter&   r  issubsetsorted
differencemaxmin)rQ   r)   Zexpected_hash_cachedirsr+   r  Zitems_to_deleteZ	nb_hashesZitems_to_delete_2048bZitems_to_delete_emptybytes_limit_too_smallZitems_to_delete_500bZitems_to_delete_6000bZsurviving_itemsr   r   r   test__get_items_to_delete  s(    
r%  c                 C   s   t | \}}}|j }|  |j }t|t|ks>J d|_|  |j }t|t|ksjJ d|_|  |j }tt|t|sJ t|dksJ d}||_|  |j }|g ksJ d S )Nr  Z3Kr   r  )	r
  r/   r0   Zreduce_sizer   bytes_limitr  r  r&   )rQ   r)   r+   Zref_cache_itemsZcache_itemsr$  r   r   r   test_memory_reduce_size  s&    




r'  c                 C   s0   t | \}}}|  t|jjg ks,J d S r7   )r
  rN   r1   rj   r/   r!   )rQ   r)   r+   r   r   r   test_memory_clear  s    r(  c                  C   s   dgd } | S )Nr  i  r   )complex_objr   r   r   fast_func_with_complex_output  s    
r*  c                 C   s"   dd t tdD }| r|S dS )Nc                 S   s   i | ]}t ||qS r   )ry   r]   r   r   r   
<dictcomp>  ra   z=fast_func_with_conditional_complex_output.<locals>.<dictcomp>g     j@zsimple output)r%   int)Zcomplex_outputr)  r   r   r   )fast_func_with_conditional_complex_output  s    r-  c                    sb   t | jd}|t tdd fddtdD  | \}}d}||vsRJ ||vs^J d S )Nr   r   r[   c                 3   s   | ]}t   V  qd S r7   r\   r]   r   r   r   r`     ra   zMtest_cached_function_race_condition_when_persisting_output.<locals>.<genexpr>r#   r   )r   rK   r$   r*  r   r%   r   rQ   Zcapfdr)   r   stderrr   r   r.  r   :test_cached_function_race_condition_when_persisting_output  s    
 r1  c                    sb   t | jd}|t tdd fddtdD  | \}}d}||vsRJ ||vs^J d S )Nr   r   r[   c                 3   s*   | ]"}t  |d  dkrdndV  qdS )r   r   TFNr\   r]   r.  r   r   r`     s   zOtest_cached_function_race_condition_when_persisting_output_2.<locals>.<genexpr>r#   r   )r   rK   r$   r-  r   r%   r   r/  r   r.  r   <test_cached_function_race_condition_when_persisting_output_2  s    
r2  c              
   C   s   t | jd}dd }||}d}||\}}||ks:J t| t||}||\}	}
t|dkshJ d}||d v s|J |	|ksJ |
|ksJ t| ||}z|  tdW n: t	y } z"d	}|t
|jv sJ W Y d }~n
d }~0 0 d S )
Nr   c                 S   s   t d | t   fS )Ng{Gz?)r   r   r  r   r   r   r'     s    
zItest_memory_recomputes_after_an_error_while_loading_results.<locals>.funcr  r   r   r   z<It normally not possible to load a corrupted MemorizedResultzis corrupted)r   rK   r$   r6   r?   r&   r   r   AssertionErrorr   ry   r   )rQ   r   r)   r'   Zcached_funcZ	input_argr  r   r   Zrecomputed_argZrecomputed_timestampr   Z	referenceerz   r   r   r   ;test_memory_recomputes_after_an_error_while_loading_results  s0    


r5  c                 C   s  t d 2}t| jdd}|jj| js,J W d    n1 s@0    Y  t|dksZJ dt|d jv spJ t d $}t }|j	d u sJ W d    n1 s0    Y  t|dksJ dt|d jv sJ d}t
t|d" t| j| jdd	}W d    n1 s0    Y  d S )
Nr   )r   r"   r   z,The 'cachedir' parameter has been deprecatedr   r   z,You set both "location='.+ and "cachedir='.+)r   )r!   r   r"   )r   r   rK   r/   r!   
startswithr&   ry   rz   r   r   r   )rQ   r.   r)   Zerror_regexr   r   r   "test_deprecated_cachedir_behaviourD  s    
2
,
r7  c                   @   s   e Zd ZdZdS )IncompleteStoreBackendzAThis backend cannot be instanciated and should raise a TypeError.N)r   rP   r   __doc__r   r   r   r   r8  [  s   r8  c                   @   sP   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd ZdS )DummyStoreBackendz(A dummy store backend that does nothing.c                 O   s   dS )zOpen an item on store.Nr   )selfr   r   r   r   r   
_open_itemc  s    zDummyStoreBackend._open_itemc                 C   s   dS )z!Check if an item location exists.Nr   r;  r!   r   r   r   _item_existsg  s    zDummyStoreBackend._item_existsc                 C   s   dS )z&Move an item from src to dst in store.Nr   )r;  srcdstr   r   r   
_move_itemk  s    zDummyStoreBackend._move_itemc                 C   s   dS )zCreate location on store.Nr   r=  r   r   r   create_locationo  s    z!DummyStoreBackend.create_locationc                 C   s   dS )z&Check if an object exists in the storeFr   r;  objr   r   r   r   s  s    zDummyStoreBackend.existsc                 C   s   dS )zClear object on storeNr   rC  r   r   r   clear_locationw  s    z DummyStoreBackend.clear_locationc                 C   s   g S )z3Returns the whole list of items available in cache.r   )r;  r   r   r   r0   {  s    zDummyStoreBackend.get_itemsc                 O   s   dS )zConfigure the storeNr   )r;  r!   r   r   r   r   r   	configure  s    zDummyStoreBackend.configureN)r   rP   r   r9  r<  r>  rA  rB  r   rE  r0   rF  r   r   r   r   r:  `  s   r:  invalid_prefixc                 C   s@   t t}t| d  W d    n1 s(0    Y  |d d S )Nz&Store backend name should be a string*r   r   r   r   )rG  r  r   r   r   (test_register_invalid_store_backends_key  s    
(rI  c                  C   s@   t t} tdd  W d    n1 s(0    Y  | d d S )NZfsz.Store backend should inherit StoreBackendBase*rH  r  r   r   r   +test_register_invalid_store_backends_object  s    
(rK  c                  C   sB   t t} tddd W d    n1 s*0    Y  | d d S )Nz/tmp/joblibunknown)r!   backendzUnknown location*)r   	TypeErrorr   r   rJ  r   r   r   !test_memory_default_store_backend  s    
*rO  c                  C   sf   G dd d} |  }t t}td|d W d    n1 s>0    Y  d}|t|d jv sbJ d S )Nc                   @   s   e Zd ZdS )zHtest_warning_on_unknown_location_type.<locals>.NonSupportedLocationClassNr   r   r   r   r   NonSupportedLocationClass  s   rP  localr   zbInstanciating a backend using a NonSupportedLocationClass as a location is not supported by joblibr   )r   UserWarningr   ry   rz   )rP  Zunsupported_locationr}   Zexpected_mesager   r   r   %test_warning_on_unknown_location_type  s    
*rS  c                  C   sb   d} t | t | tft v s"J tt}t| d W d    n1 sJ0    Y  |d d S )NZisbZfake_locationzNCan't instantiate abstract class IncompleteStoreBackend with abstract methods*)r   r8  r	   r  r   rN  r   r   )backend_namer  r   r   r   )test_instanciate_incomplete_store_backend  s    

(rU  c                  C   s>   d} t | t | tft v s"J t| d}t|ts:J d S )NZdsbZdummy_location)r   r:  r	   r  r   r   )rT  backend_objr   r   r   test_dummy_store_backend  s
    

rW  c                  C   s&   t d} td| }|jdks"J d S )NZsome_folderrQ  )pathlibPathr   r!   )r2   rV  r   r   r   0test_instanciate_store_backend_with_pathlib_path  s    

rZ  c                 C   sj   d}t  }|jd u sJ t| t||jd dks8J || j t||j| jdks^J t| d S )Nz-FileSystemStoreBackend(location="{location}")r   )r   r!   r   ry   rc   rF  rK   )rQ   Zrepr_patternrM  r   r   r   "test_filesystem_store_backend_repr  s    r[  c                 C   s   dd }t | jdd}||}d}t||j||jjdksBJ |dd}d}t||j|jj|j|j	d	ksvJ t|d
j|jjdksJ d S )Nc                 S   s   | | S r7   r   r   r   r   r   my_func  s    z)test_memory_objects_repr.<locals>.my_funcr   r    z/MemorizedFunc(func={func}, location={location}))r'   r!   *   zJMemorizedResult(location="{location}", func="{func}", args_id="{args_id}"))r!   r'   r   zMemory(location={location})r   )
r   rK   r$   ry   rc   r/   r!   r   r   r   )rQ   r\  r)   Zmemorized_funcZmemorized_func_reprmemorized_resultZmemorized_result_reprr   r   r   test_memory_objects_repr  s$    



r_  c                 C   s   t | jd}|jdd }|d}t|}t|}|jj|jjksLJ |j	|j	ks\J |j
|j
kslJ t|t|ksJ d S )Nr   c                 S   s   | d S r   r   rt   r   r   r   rR     s    z'test_memorized_result_pickle.<locals>.grb   )r   rK   r$   r   r   r   r   r/   r!   r'   r   ry   )rQ   r)   rR   r^  Zmemorized_result_pickleZmemorized_result_loadsr   r   r   test_memorized_result_pickle  s    



r`  c                 C   sh   |d u rg }t | }t |}t| t| ks8J | D ]"}||v rNq@|| || ks@J q@d S r7   )varsr  keys)leftrightignored_attrsZ	left_varsZ
right_varsattrr   r   r   compare  s    rg  memory_kwargsr#   r   )rG   r"   r   g    .AZ	parameterZunused)rF   r"   r&  Zbackend_optionsc                 C   s  t f d| ji|}tt|}t|j|j t||tg dd t|t|ks\J |	t
}tt|}t|j|j t||tg dd t|t|ksJ |d}tt|}t|j|j t||tg dd t|t|ksJ d S )Nr!   )r/   r   Z_func_code_id)re  r   )r   rK   r   r   r   rg  r/   r  r   r$   r   r   )rQ   rh  r)   Zmemory_reloadedr   Zfunc_cached_reloadedr^  Zmemorized_result_reloadedr   r   r   test_memory_pickle_dump_load  s.    




ri  )r   )rE   )T)N)pr9  r   r   rL   r1   Zos.pathrX  r   r   r   r  re   r   Zjoblib.memoryr   r   r   r   r   r   r   r	   r
   r   r   Zjoblib.parallelr   r   Zjoblib._store_backendsr   r   Zjoblib.test.commonr   r   r   Zjoblib.testingr   r   r   Zjoblib.hashingr   r   r,   r6   r?   rU   rl   rn   rr   rw   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r,  floatr   r  r  r
  r  r%  r'  r(  r*  r-  r1  r2  r5  r7  r8  r:  dictrJ   rI  rK  rO  rS  rU  rW  rZ  r[  r_  r`  rg  Zmarkri  r   r   r   r   <module>   s   
+
b


'
$$ 8
!


/$

