B
    |bc8                 @   s   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ edd	Zd
d ZG dd dedZG dd deZG dd deeZdS )z.Storage providers backends for Memory caching.    N)ABCMetaabstractmethod   )concurrency_safe_rename)mkdirpmemstr_to_bytes
rm_subdirs)numpy_pickleCacheItemInfozpath size last_accessc             C   s,   t t }d||t }|| | |S )z>Writes an object into a unique file in a concurrency-safe way.z{}.thread-{}-pid-{})id	threadingcurrent_threadformatosgetpid)Zobject_to_writefilename
write_func	thread_idtemporary_filename r   Y/home/ankuromar296_gmail_com/.local/lib/python3.7/site-packages/joblib/_store_backends.pyconcurrency_safe_write   s
    
r   c               @   sp   e Zd ZdZdZedd Zedd Zedd Zed	d
 Z	edd Z
edd Zede fddZdS )StoreBackendBasezaHelper Abstract Base Class which defines all methods that
       a StorageBackend must implement.Nc             C   s   dS )a  Opens an item on the store and return a file-like object.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        f: a file-like object
            The file-like object where an item is stored and retrieved
        mode: string, optional
            the mode in which the file-like object is opened allowed valued are
            'rb', 'wb'

        Returns
        -------
        a file-like object
        Nr   )selffmoder   r   r   
_open_item'   s    zStoreBackendBase._open_itemc             C   s   dS )a  Checks if an item location exists in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        location: string
            The location of an item. On a filesystem, this corresponds to the
            absolute path, including the filename, of a file.

        Returns
        -------
        True if the item exists, False otherwise
        Nr   )r   locationr   r   r   _item_exists:   s    zStoreBackendBase._item_existsc             C   s   dS )a-  Moves an item from src to dst in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        src: string
            The source location of an item
        dst: string
            The destination location of an item
        Nr   )r   srcdstr   r   r   
_move_itemK   s    zStoreBackendBase._move_itemc             C   s   dS )zCreates a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory.
        Nr   )r   r   r   r   r   create_locationY   s    	z StoreBackendBase.create_locationc             C   s   dS )zClears a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory or a filename absolute path
        Nr   )r   r   r   r   r   clear_locationd   s    	zStoreBackendBase.clear_locationc             C   s   dS )zReturns the whole list of items available in the store.

        Returns
        -------
        The list of items identified by their ids (e.g filename in a
        filesystem).
        Nr   )r   r   r   r   	get_itemso   s    zStoreBackendBase.get_itemsr   c             C   s   dS )a  Configures the store.

        Parameters
        ----------
        location: string
            The base location used by the store. On a filesystem, this
            corresponds to a directory.
        verbose: int
            The level of verbosity of the store
        backend_options: dict
            Contains a dictionnary of named paremeters used to configure the
            store backend.
        Nr   )r   r   verbosebackend_optionsr   r   r   	configurey   s    zStoreBackendBase.configure)__name__
__module____qualname____doc__r   r   r   r   r!   r"   r#   r$   dictr'   r   r   r   r   r   !   s   
r   )	metaclassc               @   s   e Zd ZdZd&ddZd'ddZdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zd(ddZdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% ZdS ))StoreBackendMixina  Class providing all logic for managing the store in a generic way.

    The StoreBackend subclass has to implement 3 methods: create_location,
    clear_location and configure. The StoreBackend also has to provide
    a private _open_item, _item_exists and _move_item methods. The _open_item
    method has to have the same signature as the builtin open and return a
    file-like object.
    r   Nc       	   	   C   s   t jj| jf| }|dkrD|dk r4td| ntd|| t| dsRdn| j}t j|d}| |s|t	d| |dkr| 
|d	}t|}W dQ R X ntj||d
}|S )zKLoad an item from the store given its path as a list of
           strings.r   
   z{0}...z{0} from {1}	mmap_modeNz
output.pklzANon-existing item (may have been cleared).
File %s does not existrb)r0   )r   pathjoinr   printr   hasattrr0   r   KeyErrorr   r	   load)	r   r2   r%   msg	full_pathr0   r   r   itemr   r   r   	load_item   s     
zStoreBackendMixin.load_itemc                sz   yht jj jf| } |s* | t j|d}|dkrLtd|   fdd} ||| W n   Y nX dS )zLDump an item in the store at the path given as a list of
           strings.z
output.pklr/   zPersisting in %sc          	      s.     |d}tj| | jd W d Q R X d S )Nwb)compress)r   r	   dumpr=   )to_writedest_filenamer   )r   r   r   r      s    z/StoreBackendMixin.dump_item.<locals>.write_funcN)r   r2   r3   r   r   r"   r4   _concurrency_safe_write)r   r2   r:   r%   	item_pathr   r   r   )r   r   	dump_item   s    

zStoreBackendMixin.dump_itemc             C   s,   t jj| jf| }| |r(| | dS )z7Clear the item at the path, given as a list of strings.N)r   r2   r3   r   r   r#   )r   r2   rB   r   r   r   
clear_item   s    
zStoreBackendMixin.clear_itemc             C   s,   t jj| jf| }t j|d}| |S )zLCheck if there is an item at the path, given as a list of
           stringsz
output.pkl)r   r2   r3   r   r   )r   r2   rB   r   r   r   r   contains_item   s    zStoreBackendMixin.contains_itemc             C   s   dt jj| jf| iS )zReturn information about item.r   )r   r2   r3   r   )r   r2   r   r   r   get_item_info   s    zStoreBackendMixin.get_item_infoc          	   C   s^   yNt jj| jf| }t j|d}| |d}t| dS Q R X W n
   i S dS )z"Return actual metadata of an item.zmetadata.jsonr1   zutf-8N)	r   r2   r3   r   r   jsonloadsreaddecode)r   r2   rB   r   r   r   r   r   get_metadata   s    zStoreBackendMixin.get_metadatac                s\   yJt jj jf| } | t j|d} fdd} ||| W n   Y nX dS )z Store metadata of a computation.zmetadata.jsonc          	      s2     |d}|t| d W d Q R X d S )Nr<   zutf-8)r   writerG   dumpsencode)r?   r@   r   )r   r   r   r      s    z4StoreBackendMixin.store_metadata.<locals>.write_funcN)r   r2   r3   r   r"   rA   )r   r2   metadatarB   r   r   r   )r   r   store_metadata   s    
z StoreBackendMixin.store_metadatac             C   s   t jj| jf| }| |S )z,Check cached function is available in store.)r   r2   r3   r   Zobject_exists)r   r2   	func_pathr   r   r   contains_path   s    zStoreBackendMixin.contains_pathc             C   s,   t jj| jf| }| |r(| | dS )z0Clear all items with a common path in the store.N)r   r2   r3   r   r   r#   )r   r2   rQ   r   r   r   
clear_path   s    
zStoreBackendMixin.clear_pathc          	   C   sj   t jj| jf| }| |s(| | |dk	rft j|d}| |d}||d W dQ R X dS )z&Store the code of the cached function.Nzfunc_code.pyr<   zutf-8)	r   r2   r3   r   r   r"   r   rL   rN   )r   r2   	func_coderQ   r   r   r   r   r   store_cached_func_code   s    

z(StoreBackendMixin.store_cached_func_codec          	   C   sX   |dg7 }t jj| jf| }y&| |d}| dS Q R X W n    Y nX dS )z&Store the code of the cached function.zfunc_code.pyr1   zutf-8N)r   r2   r3   r   r   rI   rJ   )r   r2   r   r   r   r   r   get_cached_func_code  s    
z&StoreBackendMixin.get_cached_func_codec             C   s   dt jj| jf| iS )z?Return information related to the cached function if it exists.r   )r   r2   r3   r   )r   r2   r   r   r   get_cached_func_info  s    z&StoreBackendMixin.get_cached_func_infoc             C   s   |  | j dS )zClear the whole store content.N)r#   r   )r   r   r   r   clear  s    zStoreBackendMixin.clearc          	   C   sZ   |  |}xJ|D ]B}| jdkr,td| y| |j W q tk
rP   Y qX qW dS )z9Reduce store size to keep it under the given bytes limit.r/   zDeleting item {0}N)_get_items_to_deleter%   r4   r   r#   r2   OSError)r   bytes_limititems_to_deleter:   r   r   r   reduce_store_size  s    


z#StoreBackendMixin.reduce_store_sizec             C   s   t |trt|}|  }tdd |D }|| }|dk r@g S |jtdd g }d}x*|D ]"}||krnP || ||j	7 }q`W |S )z9Get items to delete to keep the store under a size limit.c             s   s   | ]}|j V  qd S )N)size).0r:   r   r   r   	<genexpr>-  s    z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>r   last_access)key)

isinstancestrr   r$   sumsortoperator
attrgetterappendr^   )r   r[   itemsr^   Zto_delete_sizer\   Zsize_so_farr:   r   r   r   rY   '  s     


z&StoreBackendMixin._get_items_to_deletec             C   s   t |||}| || dS )z7Writes an object into a file in a concurrency-safe way.N)r   r!   )r   r?   r   r   r   r   r   r   rA   C  s    z)StoreBackendMixin._concurrency_safe_writec             C   s   dj | jj| jdS )z/Printable representation of the store location.z#{class_name}(location="{location}"))
class_namer   )r   	__class__r(   r   )r   r   r   r   __repr__I  s    zStoreBackendMixin.__repr__)r   N)r   )N)r(   r)   r*   r+   r;   rC   rD   rE   rF   rK   rP   rR   rS   rU   rV   rW   rX   r]   rY   rA   rm   r   r   r   r   r.      s$   




r.   c               @   sN   e Zd ZdZeeZeejj	Z
eeZdd Zdd Zdd Zdd
dZd	S )FileSystemStoreBackendz7A StoreBackend used with local or network file systems.c             C   s&   || j krt| ntj|dd dS )zDelete location on store.T)ignore_errorsN)r   r   shutilrmtree)r   r   r   r   r   r#   V  s    

z%FileSystemStoreBackend.clear_locationc             C   s   t | dS )zCreate object location on storeN)r   )r   r   r   r   r   r"   ]  s    z&FileSystemStoreBackend.create_locationc       	         s   g }xt | jD ]\ }}tdt j }|rt j d}yt j|}W n< t	k
r   yt j }W n t	k
r   wY nX Y nX t
j
|}y( fdd|D }tdd |D }W n t	k
r   wY nX |t || qW |S )z7Returns the whole list of items available in the store.z[a-f0-9]{32}z
output.pklc                s   g | ]}t j |qS r   )r   r2   r3   )r_   fn)dirpathr   r   
<listcomp>v  s   z4FileSystemStoreBackend.get_items.<locals>.<listcomp>c             s   s   | ]}t j|V  qd S )N)r   r2   getsize)r_   rr   r   r   r   r`   x  s   z3FileSystemStoreBackend.get_items.<locals>.<genexpr>)r   walkr   rematchr2   basenamer3   getatimerZ   datetimefromtimestampre   ri   r
   )	r   rj   _	filenamesZis_cache_hash_dirZoutput_filenamera   Zfull_filenamesdirsizer   )rs   r   r$   a  s0    

z FileSystemStoreBackend.get_itemsr   Nc             C   sn   |dkri }|| _ tj| j s*t| j  |dd| _|d}| jr^|dk	r^tjddd || _	|| _
dS )zsConfigure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        Nr=   Fr0   zSCompressed items cannot be memmapped in a filesystem store. Option will be ignored.   )
stacklevel)r   r   r2   existsr   getr=   warningswarnr0   r%   )r   r   r%   r&   r0   r   r   r   r'     s    

z FileSystemStoreBackend.configure)r   N)r(   r)   r*   r+   staticmethodopenr   r   r2   r   r   r   r!   r#   r"   r$   r'   r   r   r   r   rn   O  s   $rn   )r+   rw   r   os.pathr{   rG   rp   r   collectionsrg   r   abcr   r   Z	backportsr   Zdiskr   r   r    r	   
namedtupler
   r   r   objectr.   rn   r   r   r   r   <module>   s*   
i F