a
    ߙfbtU                     @   s  d dl Z d dlZd dlmZmZmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZmZmZ d dlmZmZ d dlmZmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$ d dl%m&Z& e 'dZ(ej)ej*ej+ej,ej-ej.ej/ej0ej1ej2ej3ddgZ4dd Z5dd Z6dd Z7dd Z8dd Z9dd Z:dd Z;dd Z<d d! Z=d"d# Z>d$d% Z?d&d' Z@e jABd(e4d)d* ZCd+d, ZDd-d. ZEd/d0 ZFd1d2 ZGd3d4 ZHdd6d7ZIed8ejJ d9ejJ d:ejJ d;ZKed8d<gejJ d9d=gejJ d:d>gejJ d;ZLed d8gejM d<d9gejM d8ejN ZOed d8gejP d=d:gejP d?d>gejP ZQed d8gejR ejS d d8gejR ejS d@ejJ ejT ZUeeOeUdAZVed8d<gd9d=gdBdCdDdEZWed8d<gd9d=gd:d>gdFdCdDdGgdEZXeXY ZZdHeZ_[ed8d<gd9d=gd:d>gdIdJd?gejR ejS dKd@gejR ejS dLZ\ed8d<gd9d=gd:d>gdIdJd?gejR ejS dKd@gejR ejS dMdNgejJ ejT dOZ]ed8d<gd9d=gd:d>gdIdMdNgejJ ejT dPZ^edQdRgdSdTeKdUZ_e_ed8d<gej` eWeXeZe\e]e^d8d<geja d@dVgebejc dWdXgedejeejTd<   dYdZgejf ed8d<gejM ed8d<gejM d[ejM d\ed8d<gejM eLeOeQeUeVd]Zgg d^Zhd_gd_gehg d`g dag dbg dcg ddg deg dfdgdhgdgdhgdgdhgdgdhgg didgdhgdgdhgg djg d;g dkg d;g dlg dmdnZidodp Zje jABdqeefdrds Zke jABdtelegm e jABdqeefdudv Zndwdx Zoe jABdqeefdydz Zpe jABdqeefd{d| Zqe jABdqeefd}d~ Zre jABdqeefdd Zsdd Ztdd Zue jAvddd ZwdS )    N)TableQTableNdarrayMixinColumn)simple_table)units)SkyCoordLatitude	LongitudeAngleEarthLocationSphericalRepresentationCartesianRepresentationSphericalCosLatDifferential)parquet_identifyget_pyarrow)Time	TimeDelta)allclose)QuantityInfo)AstropyUserWarning)_NOT_OVERWRITING_MSG_MATCH)
HAS_PANDASpyarrow|S3U3c                 C   s>   | t jkrg dS | dkr"g dS | dkr2g dS g dS d S )N)r      r   r   )s   abcs   defs   ghir   )abcdefZghir         )npbool_dtype r&   Alib/python3.9/site-packages/astropy/io/misc/tests/test_parquet.py_default_values   s    
r(   c                 C   sX   |  d}t }|tdg dd || t|}t|d g dksTJ dS )z+Test writing/reading a simple parquet file.test.parquetar   namedataN)joinr   
add_columnr   writereadr"   alltmpdir	test_filet1t2r&   r&   r'   test_read_write_simple)   s    


r8   c                 C   s   |  d}t|d}W d   n1 s*0    Y  t }|tdg dd tjttd |	| W d   n1 s~0    Y  dS )z2Test writing an existing file without overwriting.r)   wNr*   r   r+   match)
r.   openr   r/   r   pytestraisesOSErrorr   r0   )r4   r5   fr6   r&   r&   r'   test_read_write_existing3   s    
rA   c                 C   s   |  d}t|d}W d   n1 s*0    Y  t }|tdg dd |j|dd t|}t|d g dksJ dS )	z"Test overwriting an existing file.r)   r9   Nr*   r   r+   TZ	overwrite)	r.   r<   r   r/   r   r0   r1   r"   r2   )r4   r5   r@   r6   r7   r&   r&   r'   "test_read_write_existing_overwrite?   s    

rC   c                 C   s   |  d}t }|tdg dd || ddl}|j|dd4}t|}t	|d g dkslJ W d   n1 s0    Y  dS )	zTest reading a file object.r)   r*   r   r+   r   Nr)mode)
r.   r   r/   r   r0   ioFileIOr1   r"   r2   )r4   r5   r6   rF   Z
input_filer7   r&   r&   r'   test_read_fileobjL   s    


rH   c                 C   sj   |  d}t }|tdg dd || ddl}||}t|}t	|d g dksfJ dS )z Test reading a path-like object.r)   r*   r   r+   r   N)
r.   r   r/   r   r0   pathlibPathr1   r"   r2   )r4   r5   r6   rI   pr7   r&   r&   r'   test_read_pathlikeobj[   s    



rL   c                  C   sT   G dd d} |  }t jtdd tj|dd W d   n1 sF0    Y  dS )z*Test reading an incorrect fileobject type.c                   @   s   e Zd Zdd ZdS )z)test_read_wrong_fileobj.<locals>.FakeFilec                 S   s   d S Nr&   selfr&   r&   r'   not_readn   s    z2test_read_wrong_fileobj.<locals>.FakeFile.not_readN__name__
__module____qualname__rP   r&   r&   r&   r'   FakeFilem   s   rU   z5pyarrow can only open path-like or file-like objects.r:   parquetformatN)r=   r>   	TypeErrorr   r1   rU   r@   r&   r&   r'   test_read_wrong_fileobjj   s    r[   c                  C   s(   G dd d} |  }t dd|r$J dS )z&Test identifying an incorrect fileobj.c                   @   s   e Zd Zdd ZdS )z-test_identify_wrong_fileobj.<locals>.FakeFilec                 S   s   d S rM   r&   rN   r&   r&   r'   rP   |   s    z6test_identify_wrong_fileobj.<locals>.FakeFile.not_readNrQ   r&   r&   r&   r'   rU   {   s   rU   testNr   rZ   r&   r&   r'   test_identify_wrong_fileobjx   s    r^   c                   C   s   t dddrJ dS )(Test identifying an incorrect extension.r\   ztest.notparquetNr]   r&   r&   r&   r'   "test_identify_file_wrong_extension   s    r`   c                   C   s$   t dddsJ t ddds J dS )r_   r\   r)   Nz	test.parqr]   r&   r&   r&   r'   $test_identify_file_correct_extension   s    ra   c                   C   s   t dddrJ dS )z-Test running identify with no object or path.r\   Nr]   r&   r&   r&   r'   "test_identify_file_noobject_nopath   s    rb   c                  C   s\   t  } | tdg dd tjtdd | jddd W d	   n1 sN0    Y  d	S )
z-Test writing to a filename of the wrong type.r*   r   r+   zshould be a stringr:   i  rV   rW   N)r   r/   r   r=   r>   rY   r0   r6   r&   r&   r'   test_write_wrong_type   s    rd   r%   c                 C   st   |  d}t|}t }|tdtj||dd || t|}t	|d |ks^J |d j
|kspJ dS )z=Test that round-tripping a single column preserves datatypes.r)   r*   r$   r+   N)r.   r(   r   r/   r   r"   arrayr0   r1   r2   r%   )r4   r%   r5   valuesr6   r7   r&   r&   r'   test_preserve_single_dtypes   s    


rg   c              	   C   s   |  d}t }tD ],}t|}|tt|tj||dd q|	| t
|}tD ]<}t|}t|t| |ksJ |t| j|ksZJ qZdS )zBTest that round-tripping preserves a table with all the datatypes.r)   r$   r+   N)r.   r   
ALL_DTYPESr(   r/   r   strr"   re   r0   r1   r2   r%   )r4   r5   r6   r%   rf   r7   r&   r&   r'   test_preserve_all_dtypes   s    
"

rj   c                 C   s   |  d}t }|tdg dd d|jd< d|jd< d|jd	< d
|jd< tg d|jd< || t|}|jD ]"}t	|j| |j| ks|J q|dS )z-Test that writing/reading preserves metadata.r)   r*   r   r+   r   Zhellobgn!	@cTdeN)
r.   r   r/   r   metar"   re   r0   r1   r2   )r4   r5   r6   r7   keyr&   r&   r'   test_preserve_meta   s    







rq   c                 C   s  |  d}t }tg ddd|d< d|d jd< dd	d
gi|d jd< d|d _d|d _d
|jd< dd	d
gi|jd< |j|dd t|}|d j|d jksJ |d j|d jksJ |d j|d jksJ |d j|d jksJ |j|jksJ dS )z<Test that writing/reading preserves unit/format/description.r)   r   sr-   unitr*   A0a0a1r   r   7.3fA columnrk   c0rl   TrB   N	r.   r   r   ro   rX   descriptionr0   r1   rt   r3   r&   r&   r'   test_preserve_serialized   s     




r}   c                 C   s$  |  d}t }tg ddd|d< d|d jd< dd	d
gi|d jd< d|d _d|d _d
|jd< dd	d
gi|jd< dd |jd< dd |jd< |j|dd t|}|d j|d jksJ |d j|d jksJ |d j|d jksJ |d j|d jksJ |j|jks J dS )z"Test that very large datasets workr)   r   rr   rs   r*   ru   rv   rw   r   r   rx   ry   rk   rz   rl   0i  Zmeta_bigi   Zmeta_biggerstillTrB   Nr{   r3   r&   r&   r'   test_metadata_very_large   s$    




r   c                 C   s   |  d}t }|tdg dd t|jd< tt}|	| W d   n1 sZ0    Y  dt|j
v svJ dt|j
v sJ dS )	z1Test that we cannot preserve objects in metadata.r)   r*   r   r+   r@   Nzcannot represent an objectz<class 'str'>)r.   r   r/   r   ri   ro   r=   r>   	Exceptionr0   value)r4   r5   r6   errr&   r&   r'   test_fail_meta_serialize  s    

(r   Tc           	   
   C   s   |r| j |j u sJ g d}|| D ]}| }|}|dD ]@}zt||}t||}W q: tyx   || }|| }Y q:0 q:|dkr|du ri }|du ri }t|tjr|jjdkrt	||ddsJ q$t
||ks$J q$dS )z9Convenient routine to check objects and attributes match.)z	info.namezinfo.formatz	info.unitzinfo.description	info.meta.r   Nr@   gV瞯<)Zrtol)	__class__splitgetattrAttributeError
isinstancer"   ndarrayr%   kindquantity_allcloser2   )	Zobj1Zobj2attrscompare_classZ
info_attrsattrrw   Za2Zsubattrr&   r&   r'   assert_objects_equal   s*    
r   r   r!      )xyzr             
   )Zdifferentialszdeg,degZfk4zJ1990.5)rt   frameZobstimez	deg,deg,mzJ1991.5Z	cartesianz
deg,deg,pc   	   )rt   pm_ra_cosdecpm_dec      )rt   r   r   radial_velocity)rt   r   g   @BAg   BAZjdZtai)rX   scalelocation   g      @g      @      g     f@)
wrap_angle)tmdtscscdscdcscpmscpmrvscrvr   qdbqdexqmaglatlonangel2srcrsdsrd)r   shaperX   r   r   r-   )r   r   rX   r   )radecrepresentation_type
frame.name)r   r   distancer   r   )r   r   r   r   r   )r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   rt   )r   rt   r   )r   r   r   Z	ellipsoid)r   r   r   )Zd_lon_coslatZd_latZ
d_distance)r   r   r   zdifferentials.s.d_lon_coslatzdifferentials.s.d_latzdifferentials.s.d_distance)c1c2r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zndr   r   r   r   c           
      C   s   |  d}tt}tdd |D |d}|j|dd tj|dd}|j|jksTJ |j	 D ]v\}}|| }t
|tr|j|_t| }d}	t
|jtrt|tu sJ dg}d	}	t|j|ksJ t||||	 q^d
S )zWTest writing as QTable and reading as Table.  Ensure correct classes
    come out.
    test_simple.parquetc                 S   s   g | ]}t | qS r&   
mixin_cols.0r,   r&   r&   r'   
<listcomp>      z7test_parquet_mixins_qtable_to_table.<locals>.<listcomp>namesrV   rW   Trt   FN)r.   sortedr   r   r0   r   r1   colnamescolumnsitemsr   r   rX   compare_attrsinfor   typer   r"   r2   r   r   )
r4   filenamer   tr7   r,   colZcol2r   r   r&   r&   r'   #test_parquet_mixins_qtable_to_table  s$    

r   	table_clsc                 C   s   | d}tt}| dd |D |d}d|jd< g d|jd< g d	|jd
< |j|dd | j|dd}|jd dks|J |jd g dksJ |jd
 g d	ksJ |j|jksJ dS )zGTest write/read all cols at once and validate intermediate column namesr   c                 S   s   g | ]}t | qS r&   r   r   r&   r&   r'   r     r   z.test_parquet_mixins_as_one.<locals>.<listcomp>r   spamCthisisr*   ZcommentcommentsfirstsecondZthirdhistoryrV   rW   Nr.   r   r   ro   r0   r1   r   r   r4   r   r   r   r7   r&   r&   r'   test_parquet_mixins_as_one  s    

r   name_colc           
      C   s  | d}|\}}ddg}| |||gd|dgd}d|| j_ttdd	d
id|| j_|jsltd t	|| t
rtd |j|dd | j|dd}|j|jksJ |jD ]}	t||	 ||	 t|	  q|dr|| jjjtju sJ || jjjtju sJ dS )z<Test write/read one col at a time and do detailed validationr   g      ?g       @r   r   r   zmy description2   r*   Zbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)listdictz7column is not a mixin (e.g. Quantity subclass in Table)zNdarrayMixin not supportedrV   rW   r   N)r.   r   r|   r   rangero   Zhas_mixin_columnsr=   skipr   r   Zxfailr0   r1   r   r   r   
startswithZ_timeZjd1r   r"   r   Zjd2)
r   r   r4   r   r,   r   rl   r   r7   Zcolnamer&   r&   r'   test_parquet_mixins_per_column  s$    



r   c                 C   s   |  d}tdd}g d|d< d|d jd< |j|dd t|}|jd	u sTJ |j|jksdJ |jD ]j}t	|| j|| jksJ t	|| || ksJ d	|| _d	|| _t	|| || ksjJ qjd
S )aK  Test round-trip of MaskedColumn through Parquet using default serialization
    that writes a separate mask column.  Note:

    >>> simple_table(masked=True)
    <Table masked=True length=3>
      a      b     c
    int64 float64 str1
    ----- ------- ----
       --     1.0    c
        2     2.0   --
        3      --    e
    r)   T)masked)   c   d   erl   r   rV   rW   FN)
r.   r   maskr0   r   r1   r   r   r"   r2   )r4   r   r   r7   r,   r&   r&   r'   $test_round_trip_masked_table_default  s    





r   c                 C   s   | d}tt}| dd |D |d}d|jd< g d|jd< g d	|jd
< |j|dd |D ]d}| j|d|gd}|jd dksJ |jd g dksJ |jd
 g d	ksJ |j|gks`J q`dS )z4Test write all cols at once, and read one at a time.r   c                 S   s   g | ]}t | qS r&   r   r   r&   r&   r'   r     r   z5test_parquet_mixins_read_one_name.<locals>.<listcomp>r   r   r   r   r   r   r   rV   rW   rX   Zinclude_namesNr   )r   r4   r   r   r   r,   r7   r&   r&   r'   !test_parquet_mixins_read_one_name  s    

r   c                 C   s   | d}tt}| dd |D |d}d|jd< g d|jd< g d	|jd
< |j|dd | j|d|dd d}|jdd |jksJ dS )z<Test write all cols at once, and read all but one at a time.r   c                 S   s   g | ]}t | qS r&   r   r   r&   r&   r'   r   -  r   z:test_parquet_mixins_read_exclude_names.<locals>.<listcomp>r   r   r   r   r   r   r   rV   rW   r   r   )rX   Zexclude_namesNr   r   r&   r&   r'   &test_parquet_mixins_read_exclude_names'  s    

r   c                 C   s   | d}tt}| dd |D |d}d|jd< g d|jd< g d	|jd
< |j|dd tjtdd$ | j|dddgd}W d   n1 s0    Y  dS )z>Test write all cols at once, and try to read no valid columns.r   c                 S   s   g | ]}t | qS r&   r   r   r&   r&   r'   r   >  r   z7test_parquet_mixins_read_no_columns.<locals>.<listcomp>r   r   r   r   r   r   r   rV   rW   zNo include_names specifiedr:   Znot_a_columnZalso_not_a_columnr   N)	r.   r   r   ro   r0   r=   r>   
ValueErrorr1   r   r&   r&   r'   #test_parquet_mixins_read_no_columns8  s    

r   c                 C   s   | d}tt}| dd |D |d}d|jd< g d|jd< g d	|jd
< |j|dd | j|ddd}|jd dks~J |jd g dksJ |jd
 g d	ksJ |j|jksJ t|dksJ dS )z1Test write all cols at once, and read the schema.r   c                 S   s   g | ]}t | qS r&   r   r   r&   r&   r'   r   P  r   z3test_parquet_mixins_read_schema.<locals>.<listcomp>r   r   r   r   r   r   r   rV   rW   T)rX   Zschema_onlyr   N)r.   r   r   ro   r0   r1   r   lenr   r&   r&   r'   test_parquet_mixins_read_schemaJ  s    

r   c                 C   s   |  d}t }ttdtjd|d< ttdddtjd|d< |j|dd	 tj|d
gd}|d 	 dk svJ tj|dgd}|d 	 dk sJ dS )z*Test reading a parquet file with a filter.r   d   )r-   r%   r*   r   rk   TrB   )r*   <r   )filtersr   )rk   r   r   N)
r.   r   r   r"   Zarangeint32float64r0   r1   max)r4   r   r6   r7   r&   r&   r'   test_parquet_filterb  s    
r  c              	      sN  |  d}t  tD ],}t|} tt|tj||dd q j	j
} fdd|D }t|}t \}}}	|j|||	d8}
 fdd|D }|
tjj||d W d	   n1 s0    Y  tjtd
d t|}W d	   n1 s0    Y  tD ]B}t|}t|t| |ks.J |t| j	|ksJ qd	S )z$Test reading a generic parquet file.ztest_generic.parqr$   r+   c                    s"   g | ]}|t  | jjfqS r&   )r   Zfrom_numpy_dtyper%   r   r   rc   r&   r'   r     s   z-test_parquet_read_generic.<locals>.<listcomp>versionc                    s   g | ]}t  | jqS r&   )r   re   r-   r   rc   r&   r'   r     s   )schemaNNo table::lenr:   )r.   r   rh   r(   r/   r   ri   r"   re   r%   r   r   r  r   ZParquetWriterZwrite_tableZfrom_arraysr=   warnsr   r1   r2   )r4   r   r%   rf   r   Z	type_listr  _rV   writer_versionwriterZarraysr7   r&   rc   r'   test_parquet_read_genericu  s,    
"


4(r
  znot HAS_PANDASc           	   	   C   s   |  d}t }tD ],}t|}|tt|tj||dd q|	 }t
 \}}}|j||d tjtdd t|}W d   n1 s0    Y  tD ]<}t|}t|t| |ksJ |t| j|ksJ qdS )z#Test reading a pandas parquet file.ztest_pandas.parqr$   r+   r  r  r:   N)r.   r   rh   r(   r/   r   ri   r"   re   Z	to_pandasr   Z
to_parquetr=   r  r   r1   r2   r%   )	r4   r   r6   r%   rf   Zdfr  r  r7   r&   r&   r'   test_parquet_read_pandas  s    
"(r  )T)xr=   Znumpyr"   Zastropy.tabler   r   r   r   Zastropy.table.table_helpersr   Zastropyr   uZastropy.coordinatesr   r	   r
   r   r   r   r   r   Zastropy.io.misc.parquetr   r   Zastropy.timer   r   Zastropy.unitsr   r   Zastropy.units.quantityr   Zastropy.utils.exceptionsr   Zastropy.utils.miscr   Z"astropy.utils.compat.optional_depsr   Zimportorskipr   Zuint8Zuint16Zuint32Zuint64Zint8Zint16r   Zint64Zfloat32r   r#   rh   r(   r8   rA   rC   rH   rL   r[   r^   r`   ra   rb   rd   ZmarkZparametrizerg   rj   rq   r}   r   r   r   ZkmZelr   ZdegZkpcr   Zpcr   ZmasZyrrr   r   r   r   r   copyr   r   r   r   r   r   ZdaymZdBZmWZdexcmZABmagr   Z
time_attrsr   r   r   r   r   r   r   r   r   r   r   r  r
  Zskipifr  r&   r&   r&   r'   <module>   s
  (

	

$ , $2$$%
!



 
