a
    GGb-                     @   s(  d Z ddlmZmZ ddlZddlZddlZddlm	Z	 ddl
m  mZ ddlmZ ddlmZ ddlm  mZ ddlmZ ddlmZ ejjejdd	d
ejdddgejdddgdd Zejjejdd	d
dd Zejdd Z ejj!ej"de# G dd dZ$dS )zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
    )BytesIOStringION)is_ci_environment)	DataFrame)read_featherread_csvVhttps://github.com/pandas-dev/pandas/raw/main/pandas/tests/io/parser/data/salaries.csvT)urlZcheck_before_testmodeexplicitinferenginepythoncc                 C   sB   t j| }d}|| }|dkr"|}t|d||d}t||  d S )Nr	   r   	)sepcompressionr   )icomZ_compression_to_extensionr   tmassert_frame_equal)Zsalaries_tabler   r   Zcompression_only	extensionZbase_urlr
   Z	url_table r   Blib/python3.9/site-packages/pandas/tests/io/parser/test_network.pytest_compressed_urls   s    
r   ghttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/parser/data/unicode_series.csvc                  C   s(   d} t | ddd}|jd dks$J dS )zN
    read_csv should honor the requested encoding for URLs.

    GH 10424
    r   zlatin-1N)encodingheader)      u$   Á köldum klaka (Cold Fever) (1994))r   Zloc)pathdfr   r   r   test_url_encoding_csv5   s    r"   c                 C   s   t | ddddS )z DataFrame with the tips dataset.iodataZcsvtips.csvr   )datapathr   r   r   tips_dfK   s    r'   s3_resourcec                   @   s   e Zd Zeddd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd Zejjddddd Zejjdddeddd Zejjdd Zejjejje d d!d"d# Zd$d% Zedd&d' Zd(S ))TestS3s3fsc                 C   sx   dD ]>\}}t d| ||d}t|ts,J |jr6J t|| qt d|d}t|ts^J |jrhJ t|| d S )N) N).gzZgzip).bz2bz2s3://pandas-test/tips.csv)r   storage_optionszs3://cant_get_it/tips.csvr1   r   
isinstancer   emptyr   r   selfr'   s3soextcompr!   r   r   r   test_parse_public_s3_bucketU   s    

z"TestS3.test_parse_public_s3_bucketc                 C   s@   t dd|d}t|tsJ |jr&J t|jd d | d S )Nzs3n://pandas-test/tips.csv
   nrowsr1   r   r4   r   r5   r   r   ilocr7   r'   r8   r!   r   r   r   test_parse_public_s3n_bucketj   s    
z#TestS3.test_parse_public_s3n_bucketc                 C   s@   t dd|d}t|tsJ |jr&J t|jd d | d S )Nzs3a://pandas-test/tips.csvr<   r=   r?   rA   r   r   r   test_parse_public_s3a_bucketr   s    
z#TestS3.test_parse_public_s3a_bucketc                 C   sT   dD ]J\}}t d| d||d}t|ts.J |jr8J t|jd d | qd S )Nr+   r0   r<   )r>   r   r1   r?   r6   r   r   r   !test_parse_public_s3_bucket_nrowsy   s    
z(TestS3.test_parse_public_s3_bucket_nrowsc           
   	   C   s   d}dD ]\}}t d| |||dn}|j|ks4J dD ]J}| }t|tsRJ |jr\J |j|| ||d   }	t|	| q8W d    q1 s0    Y  qd S )N   r+   r0   )	chunksizer   r1   r   r      r   	r   rF   Z	get_chunkr4   r   r5   r@   r   r   
r7   r'   r8   rF   r9   r:   Z	df_readerZi_chunkr!   Ztrue_dfr   r   r   #test_parse_public_s3_bucket_chunked   s$    
z*TestS3.test_parse_public_s3_bucket_chunkedc           
   	   C   s   d}dD ]\}}t d| ||d|dn}|j|ks6J dD ]J}| }t|tsTJ |jr^J |j|| ||d   }	t|	| q:W d    q1 s0    Y  qd S )NrE   r+   r0   r   )rF   r   r   r1   rG   r   rI   rJ   r   r   r   *test_parse_public_s3_bucket_chunked_python   s&    
z1TestS3.test_parse_public_s3_bucket_chunked_pythonc                 C   sJ   dD ]@\}}t d| d||d}t|ts.J |jr8J t|| qd S )Nr+   r0   r   r   r   r1   r3   r6   r   r   r   "test_parse_public_s3_bucket_python   s    
z)TestS3.test_parse_public_s3_bucket_pythonc                 C   sF   dD ]<}t d| dd|d}t|ts*J |jr4J t|| qd S )N)r,   r-   r.   r0   r   r   rM   r3   )r7   r'   r8   r9   r!   r   r   r   test_infer_s3_compression   s    
z TestS3.test_infer_s3_compressionc                 C   sV   dD ]L\}}t d| dd||d}t|ts0J |jr:J t|jd d | qd S )Nr+   r0   r   r<   )r   r>   r   r1   r?   r6   r   r   r   (test_parse_public_s3_bucket_nrows_python   s    
z/TestS3.test_parse_public_s3_bucket_nrows_pythonc                 C   sx   d}t jt|d td|d W d    n1 s40    Y  t jt|d td W d    n1 sj0    Y  d S )N#The specified bucket does not existmatchzs3://nyqpug/asdf.csvr2   zs3://cant_get_it/file.csv)pytestraisesOSErrorr   )r7   r8   msgr   r   r   test_read_s3_fails   s
    *zTestS3.test_read_s3_failszGH#39155 s3fs upgradeF)reasonstrictc                 C   sT   dd l }t|jjf}tj|dd |jd|d W d    n1 sF0    Y  d S )Nr   rQ   rR   z/s3://an_s3_bucket_data_doesnt_exit/not_real.csvr2   )botocoreFileNotFoundError
exceptionsClientErrorrT   rU   to_csvr7   r'   r8   r[   errorr   r   r   test_write_s3_csv_fails   s    zTestS3.test_write_s3_csv_failsZpyarrowc                 C   sT   dd l }t|jjf}tj|dd |jd|d W d    n1 sF0    Y  d S )Nr   rQ   rR   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquetr2   )r[   r\   r]   r^   rT   rU   Z
to_parquetr`   r   r   r   test_write_s3_parquet_fails   s    z"TestS3.test_write_s3_parquet_failsc                 C   s~   |j jjddd}t|d  }t|dd}W d    n1 sD0    Y  t|ts\J |jrfJ t|}t	
|| d S )Npandas-testr%   )BucketKeyBodyutf8)r   )metaZclientZ
get_objectr   readr   r4   r   r5   r   r   )r7   r(   Z	tips_fileZ	s3_objectbufferresultexpectedr   r   r   $test_read_csv_handles_boto_s3_object  s    *
z+TestS3.test_read_csv_handles_boto_s3_objectzThis test can hang in our CI min_versions build and leads to '##[error]The runner has received a shutdown signal...' in GHA. GH: 45651)rY   c                 C   s   dd l }ttjddtdd}t }|| t|	 
d}|djd|d	 |j  |jtjd
d6 tdd|d ddd |jD v sJ W d    n1 s0    Y  d S )Nr   i    Zabcd)columnszutf-8rd   zlarge-file.csv)rf   rg   r*   )Zloggerzs3://pandas-test/large-file.csvrE   r=   )r   i  T c                 s   s   | ]}|j d d V  qdS )N)args).0xr   r   r   	<genexpr>/      z8TestS3.test_read_csv_chunked_download.<locals>.<genexpr>)r*   r   npZrandomZrandnlistr   r_   r   getvalueencodere   Z
put_objectZS3FileSystemZclear_instance_cacheZat_levelloggingDEBUGr   Zrecords)r7   r(   Zcaplogr8   r*   r!   Zstr_bufZbufr   r   r   test_read_csv_chunked_download  s    	

z%TestS3.test_read_csv_chunked_downloadc                 C   s   t d|d}t|| d S )Nzs3://pandas-test/tips#1.csvr2   )r   r   r   )r7   r'   r8   rl   r   r   r   test_read_s3_with_hash_in_key1  s    z$TestS3.test_read_s3_with_hash_in_keyc                 C   s$   t |}t d|d}t|| d S )Nz's3://pandas-test/simple_dataset.featherr2   )r   r   r   )r7   Zfeather_filer8   rm   resr   r   r   test_read_feather_s3_file_path6  s
    z%TestS3.test_read_feather_s3_file_pathN)__name__
__module____qualname__tdZ
skip_if_nor;   rB   rC   rD   rK   rL   rN   rO   rP   rX   rT   markZxfailrb   rc   
single_cpurn   Zskipifr   r}   r~   r   r   r   r   r   r)   Q   s6   



r)   )%__doc__r#   r   r   r{   Znumpyrw   rT   Zpandas.compatr   Zpandas.util._test_decoratorsutilZ_test_decoratorsr   Zpandasr   Zpandas._testingZ_testingr   Zpandas.io.commoncommonr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   r   ZnetworkZparametrizer   r"   Zfixturer'   r   ZusefixturesZskip_if_not_us_localer)   r   r   r   r   <module>   s>   

