a
    0aq                     @   s  d Z ddlZddlZddlZddlZddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZmZmZmZ dd
lm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z=m'Z' ddl>m?Z?m@Z@mAZAmBZBmCZC ddlDmEZE e=FeGZHeIdZJG dd deZKG dd deZLG dd deZMG d d! d!eZNeeLeeOePePeQf f ZRd"d#iZSdZTd$ZUd%ZVe&eQd&d'd(ZWG d)d* d*eZXe'j'j(ePeYd+d,d-ZZG d.d/ d/e0Z[G d0d1 d1Z\G d2d3 d3eZ]G d4d5 d5e:Z^e.ePeeP d6d7d8Z_e.e2dd9d:d;Z`e.eePef d<d=d>ZadS )?z
    sphinx.builders.linkcheck
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    The CheckExternalLinksBuilder class.

    :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
    :license: BSD, see LICENSE for details.
    N)datetimetimezone)parsedate_to_datetime)
HTMLParser)path)PriorityQueueQueue)Thread)AnyDict	GeneratorList
NamedTupleOptionalPatternSetTupleUnioncast)unquoteurlparse
urlunparse)nodes)Element)Response)ConnectionError	HTTPErrorTooManyRedirects)Sphinx)DummyBuilder)Config)RemovedInSphinx50Warning)BuildEnvironment)__)SphinxPostTransform)
encode_uriloggingrequests)darkgray	darkgreenpurplered	turquoise)get_node_linez([a-z]+:)?//c                   @   s*   e Zd ZU eed< eed< ee ed< dS )	HyperlinkuridocnamelinenoN)__name__
__module____qualname__str__annotations__r   int r8   r8   8lib/python3.9/site-packages/sphinx/builders/linkcheck.pyr.   /   s   
r.   c                   @   s"   e Zd ZU eed< ee ed< dS )CheckRequest
next_check	hyperlinkN)r2   r3   r4   floatr6   r   r.   r8   r8   r8   r9   r:   5   s   
r:   c                   @   s>   e Zd ZU eed< eed< eed< eed< eed< eed< dS )CheckResultr/   r0   r1   statusmessagecodeN)r2   r3   r4   r5   r6   r7   r8   r8   r8   r9   r>   :   s   
r>   c                   @   s   e Zd ZU eed< eed< dS )	RateLimitdelayr;   N)r2   r3   r4   r=   r6   r8   r8   r8   r9   rB   C   s   
rB   ZAcceptz/text/html,application/xhtml+xml;q=0.9,*/*;q=0.8   g      N@)nodereturnc                 C   s   t jdtdd t| pdS )z
    PriorityQueue items must be comparable. The line number is part of the
    tuple used by the PriorityQueue, keep an homogeneous type for comparison.
    znode_line_or_0() is deprecated.   
stacklevelr   )warningswarnr!   r-   )rE   r8   r8   r9   node_line_or_0S   s    rL   c                       s:   e Zd ZdZedd fddZeeddddZ  ZS )	AnchorCheckParserz9Specialized HTML parser that looks for a specific anchor.N)search_anchorrF   c                    s   t    || _d| _d S NF)super__init__rN   found)selfrN   	__class__r8   r9   rQ   `   s    
zAnchorCheckParser.__init__)tagattrsrF   c                 C   s.   |D ]$\}}|dv r|| j krd| _ q*qd S )N)idnameT)rN   rR   )rS   rV   rW   keyvaluer8   r8   r9   handle_starttagf   s    z!AnchorCheckParser.handle_starttag)	r2   r3   r4   __doc__r5   rQ   r
   r\   __classcell__r8   r8   rT   r9   rM   ]   s   rM   )responseanchorrF   c                 C   sP   t |}| jdddD ]*}t|tr,| }|| |jr qBq|  |jS )zReads HTML data from a response object `response` searching for `anchor`.
    Returns True if anchor was found, False otherwise.
    i   T)Z
chunk_sizeZdecode_unicode)rM   Ziter_content
isinstancebytesdecodeZfeedrR   close)r_   r`   parserchunkr8   r8   r9   check_anchorm   s    

rg   c                   @   sb  e Zd ZdZdZedZddddZee	e
 ddd	Zee	ee
ef  dd
dZee	e
 dddZeee dddZeeeef dddZeeeeeef f dddZddddZeee dddZeedddZee	e dddZeedddZ e!ddd d!Z"eeeeedd"d#d$Z#e$dd%d&d'Z%ddd(d)Z&dS )*CheckExternalLinksBuilderz+
    Checks for broken external links.
    	linkcheckzCLook for any errors in the above output or in %(outdir)s/output.txtNrF   c                 C   s8   i | _ t | _i | _i | _td t | _t	 | _
d S )Ng      @)
hyperlinksset_good_broken_redirectedsocketZsetdefaulttimeoutr   _wqueuer   _rqueuerS   r8   r8   r9   init   s    
zCheckExternalLinksBuilder.initc                 C   s.   t jd| jjdf tdd dd | jjD S )N%s.%s is deprecated.anchors_ignorerG   rH   c                 S   s   g | ]}t |qS r8   recompile.0xr8   r8   r9   
<listcomp>       z<CheckExternalLinksBuilder.anchors_ignore.<locals>.<listcomp>)rJ   rK   rU   r2   r!   configlinkcheck_anchors_ignorers   r8   r8   r9   rv      s    z(CheckExternalLinksBuilder.anchors_ignorec                 C   s.   t jd| jjdf tdd dd | jjD S )Nru   authrG   rH   c                 S   s   g | ]\}}t ||fqS r8   rw   r{   pattern	auth_infor8   r8   r9   r}      r~   z2CheckExternalLinksBuilder.auth.<locals>.<listcomp>)rJ   rK   rU   r2   r!   r   linkcheck_authrs   r8   r8   r9   r      s    zCheckExternalLinksBuilder.authc                 C   s.   t jd| jjdf tdd dd | jjD S )Nru   	to_ignorerG   rH   c                 S   s   g | ]}t |qS r8   rw   rz   r8   r8   r9   r}      r~   z7CheckExternalLinksBuilder.to_ignore.<locals>.<listcomp>)rJ   rK   rU   r2   r!   r   linkcheck_ignorers   r8   r8   r9   r      s    z#CheckExternalLinksBuilder.to_ignorec                 C   s"   t jd| jjdf tdd | jS )Nru   goodrG   rH   )rJ   rK   rU   r2   r!   rm   rs   r8   r8   r9   r      s    zCheckExternalLinksBuilder.goodc                 C   s"   t jd| jjdf tdd | jS )Nru   brokenrG   rH   )rJ   rK   rU   r2   r!   rn   rs   r8   r8   r9   r      s    z CheckExternalLinksBuilder.brokenc                 C   s"   t jd| jjdf tdd | jS )Nru   
redirectedrG   rH   )rJ   rK   rU   r2   r!   ro   rs   r8   r8   r9   r      s    z$CheckExternalLinksBuilder.redirectedc                 C   s    t jd| jjdf tdd d S )Nru   check_threadrG   rH   rJ   rK   rU   r2   r!   rs   r8   r8   r9   r      s
    z&CheckExternalLinksBuilder.check_threadr_   rF   c                 C   s:   t jd| jjdf tdd t| j| jd d i }||S )Nru   
limit_raterG   rH   )	rJ   rK   rU   r2   r!    HyperlinkAvailabilityCheckWorkerenvr   r   )rS   r_   Zworkerr8   r8   r9   r      s    
z$CheckExternalLinksBuilder.limit_ratec                 C   s"   t jd| jjdf tdd | jS )Nru   rqueuerG   rH   )rJ   rK   rU   r2   r!   rr   rS   r_   r8   r8   r9   r      s    z CheckExternalLinksBuilder.rqueuec                 C   s    t jd| jjdf tdd g S )Nru   workersrG   rH   r   r   r8   r8   r9   r      s    z!CheckExternalLinksBuilder.workersc                 C   s"   t jd| jjdf tdd | jS )Nru   wqueuerG   rH   )rJ   rK   rU   r2   r!   rq   r   r8   r8   r9   r      s    z CheckExternalLinksBuilder.wqueue)resultrF   c                 C   s  | j |jd }t||j|j|j|j|jd}| 	| |jdkrFd S |jdkr^|jdkr^d S |jrzt
jd|j|jdd |jdkr|jrt
td	|j d
 |j  nt
td	|j  n|jdkrt
td|j  | d|j||j|j n|jdkr&t
td|j |j  n|jdkr| jjsF| jjrjt
jtd|j|j|j|jfd n"t
td|j td|j   | d|j||j|jd
 |j  n |jdkrz2dtfdtfdtfdtfdtfd|j \}}W n ty   dt }}Y n0 ||d< | jjrRt
jd|j d | d |j |j|jfd n*t
|d|j |d| d |j   | d| |j||j|jd |j  ntd|j d S )N)filenamer1   r?   rA   r/   info	uncheckedworkingoldz(%16s: line %4d) T)Znonlignoredz
-ignored- z: localz
-local-   z
ok        r   zbroken link: %s (%s))locationz
broken    z - r   Zpermanentlyz
with Foundzwith See OtherZtemporarily)i-  i.  i/  i3  i4  zwith unknown codetextz
redirect  z to zredirected zUnknown status %s.)r   doc2pathr0   dictr1   r?   rA   r/   r@   write_linkstatloggerr   r(   write_entryr)   appquietZwarningiserrorwarningr#   r+   r*   r,   KeyErrorr   linkcheck_allowed_redirects
ValueError)rS   r   r   Zlinkstatr   Zcolorr8   r8   r9   process_result   sp    


 

"

z(CheckExternalLinksBuilder.process_result)whatr0   r   liner/   rF   c                 C   s   | j d||||f  d S )Nz%s:%s: [%s] %s
)txt_outfilewrite)rS   r   r0   r   r   r/   r8   r8   r9   r   +  s    z%CheckExternalLinksBuilder.write_entry)datarF   c                 C   s"   | j t| | j d d S )N
)json_outfiler   jsondumps)rS   r   r8   r8   r9   r   /  s    z(CheckExternalLinksBuilder.write_linkstatc              	   C   s   t | j| j| }td tt| jddd| _	tt| jdd.| _
|| jD ]}| | qVW d    n1 sz0    Y  W d    n1 s0    Y  | jrd| j_d S )N z
output.txtwzoutput.jsonrD   )HyperlinkAvailabilityCheckerr   r   r   r   openr   joinZoutdirr   r   checkrk   r   rn   r   Z
statuscode)rS   Zcheckerr   r8   r8   r9   finish3  s    
Hz CheckExternalLinksBuilder.finish)'r2   r3   r4   r]   rY   r#   epilogrt   propertyr   r   rv   r   r
   r   r   r   r5   r   r   r   r7   r   r   r   r   r=   r   r   r   r	   r   r   r>   r   r   r   r   r   r8   r8   r8   r9   rh      s6   	 
7rh   c                   @   sp   e Zd ZdeeeddddZddddZdddd	Ze	e
ef eeddf d
ddZe
edddZdS )r   N)r   r   builderrF   c                 C   s\   || _ || _|| _i | _g | _dd | jjD | _|rH|j| _|j	| _
nt | _t | _
d S )Nc                 S   s   g | ]}t |qS r8   rw   rz   r8   r8   r9   r}   L  r~   z9HyperlinkAvailabilityChecker.__init__.<locals>.<listcomp>)r   r   r   rate_limitsr   r   r   rr   r   rq   r   r   r   )rS   r   r   r   r8   r8   r9   rQ   A  s    
z%HyperlinkAvailabilityChecker.__init__rj   c                 C   sH   t | jjD ]6}t| j| j| j| j| j| j}|	  | j
| qd S N)ranger   linkcheck_workersr   r   r   r   r   r   startr   append)rS   Z_ithreadr8   r8   r9   invoke_threadsU  s    
z+HyperlinkAvailabilityChecker.invoke_threadsc                 C   s.   | j   | jD ]}| j ttd d qd S rO   )r   r   r   putr:   CHECK_IMMEDIATELY)rS   Z_workerr8   r8   r9   shutdown_threads]  s    

z-HyperlinkAvailabilityChecker.shutdown_threads)rk   rF   c                 c   s   |    d}| D ]H}| |jr@t|j|j|jdddV  q| jt	t
|d |d7 }qd}||k r| j V  |d7 }qb|   d S )Nr   r   r   FrD   )r   valuesis_ignored_urir/   r>   r0   r1   r   r   r:   r   r   getr   )rS   rk   Ztotal_linksr<   Zdoner8   r8   r9   r   b  s    

z"HyperlinkAvailabilityChecker.check)r/   rF   c                    s   t  fdd| jD S )Nc                 3   s   | ]}|  V  qd S r   )match)r{   patr/   r8   r9   	<genexpr>v  r~   z>HyperlinkAvailabilityChecker.is_ignored_uri.<locals>.<genexpr>)anyr   rS   r/   r8   r   r9   r   u  s    z+HyperlinkAvailabilityChecker.is_ignored_uri)N)r2   r3   r4   r"   r    rh   rQ   r   r   r   r5   r.   r   r>   r   boolr   r8   r8   r8   r9   r   @  s    "r   c                	       s^   e Zd ZdZdeeeeeee	f e
dd fddZddddZeee d	d
dZ  ZS )r   z;A worker class for checking the availability of hyperlinks.N)r   r   r   r   r   r   rF   c                    s   || _ || _|| _|| _|| _dd | j jD | _dd | j jD | _dd | j j	D | _
|rx|j| _|j| _|j| _nt | _i | _i | _t jdd d S )Nc                 S   s   g | ]}t |qS r8   rw   rz   r8   r8   r9   r}     s   z=HyperlinkAvailabilityCheckWorker.__init__.<locals>.<listcomp>c                 S   s   g | ]}t |qS r8   rw   )r{   docr8   r8   r9   r}     s   c                 S   s   g | ]\}}t ||fqS r8   rw   r   r8   r8   r9   r}     r~   T)daemon)r   r   r   r   r   r   rv   linkcheck_exclude_documentsdocuments_excluder   r   rm   rn   ro   rl   rP   rQ   )rS   r   r   r   r   r   r   rT   r8   r9   rQ   |  s,    
z)HyperlinkAvailabilityCheckWorker.__init__rj   c           
   	      s  i j jrj jd< tdfddttttf d fddtttdfdd	 tttttf d
fdd}j }z$|\}d u rW q\}}W n t	y   |\}}}Y n0 d u rqt
j}zj| j}W n ty   Y n0 |t krLtt jt|d j  q||\}}}	|dkrttd td  njt|||||	 j  qd S )Ntimeoutrj   c                     sh   t } d| j| jf d| j| jf dg}|D ]2}| jjv r0tt}| jj|  |  S q0i S )Nz%s://%sz%s://%s/*)r   schemenetlocr   linkcheck_request_headersr   DEFAULT_REQUEST_HEADERSupdate)url
candidatesuheadersr   r8   r9   get_request_headers  s    
zAHyperlinkAvailabilityCheckWorker.run.<locals>.get_request_headersc               
      s:  dv r8 dd\} }jD ]}||rd } q@qn} d }z| d W n tyh   t| } Y n0 jD ]\}}|rp qqpd } d< z|rjjrt	j
| fdj|d}|  t|t|}|sttd| nz(t	j| fdj|d}|  W nl tttfy } zLt|trH|jjd	krH t	j
| fdj|d}|  W Y d }~n
d }~0 0 W n tyj } z|jjd
krW Y d }~dS |jjd	kr|j}|d urjt|d W Y d }~dS dt|dfW  Y d }~S |jjdkr<dt|dfW  Y d }~S dt|dfW  Y d }~S W Y d }~nhd }~0  ty } zdt|dfW  Y d }~S d }~0 0 t| j}	zj|	= W n ty   Y n0 |j !d| !dkrdS |j }
|r|
d| 7 }
 | |
rdS |j"r,|j"d j}d|
|fS d|
dfS d S )N#rD   asciir   T)streamr   r   zAnchor '%s' not found)Zallow_redirectsr   r   i  i  )r   z - unauthorizedr   F)rate-limitedr   r   r   r   i  r   /r   r   r   r   )#splitrv   r   encodeUnicodeErrorr%   r   r   linkcheck_anchorsr'   r   Zraise_for_statusrg   r   	Exceptionr#   headr   r   r   ra   r_   Zstatus_coder   r   r   r:   r5   r   r   r   r   r   rstriphistory)Zreq_urlr`   Zrexr   r   r_   rR   errr;   r   new_urlrA   )allowed_redirectr   r<   kwargsrS   r/   r8   r9   	check_uri  s    





$
.$

z7HyperlinkAvailabilityCheckWorker.run.<locals>.check_uri)r   r   rF   c                    s4    j j D ]"\}}|| r||r dS qdS )NTF)r   r   itemsr   )r   r   Zfrom_urlZto_urlrs   r8   r9   r     s    z>HyperlinkAvailabilityCheckWorker.run.<locals>.allowed_redirect)r0   rF   c                    sp  j D ].}|| r|  d|j d}d|df  S qtdksLdrPdS dstrhdS tj	| }t
t|rdS d	j< d
S nPjv rdS jv rdj dfS jv rdj d j d fS tjjD ]"}  \}}}|dk r q  q|dkr8j n.|dkrN|j< n|dkrf||fj< |||fS )Nz	 matched z! from linkcheck_exclude_documentsr   r   )r   zmailto:ztel:)r   r   r   )zhttp:zhttps:r   r   )r   r   r   )r   r   r   r   r   rD   r   )r   r   r   len
startswithuri_rer   dirnamer   r   existsr   rn   rm   ro   r   r   linkcheck_retriesadd)r0   Zdoc_matcherr   Zsrcdir_r?   rA   )r   rS   r/   r8   r9   r     s@    












z3HyperlinkAvailabilityCheckWorker.run.<locals>.checkFr   z-rate limited-   z | sleeping...)r   linkcheck_timeoutr   r   r5   r7   r   r   r   r   r   r   r   r;   r   timesleepQUEUE_POLL_SECSr   r:   Z	task_doner   r   r(   r   r>   )
rS   r   Zcheck_requestr;   r0   r1   r   r?   r   rA   r8   )r   r   r   r<   r   rS   r/   r9   run  s@    &`"/




z$HyperlinkAvailabilityCheckWorker.runr   c           
      C   s  d }|j d}|rzt|}W nT tyt   zt|}W n ttfyP   Y n 0 t|}|tt	j
  }Y n0 t | }t|jj}|d u r| jj}z| j| }W n ty   t}Y n$0 |j}	d|	 }||kr|	|k r|}||krd S t | }t||| j|< |S )NzRetry-Afterg       @)r   r   r=   r   r   	TypeErrorr   Z	timestampZnowr   ZutcZtotal_secondsr  r   r   r   r   linkcheck_rate_limit_timeoutr   r   DEFAULT_DELAYrC   rB   )
rS   r_   r;   Zretry_afterrC   Zuntilr   Z	max_delayZ
rate_limitZlast_wait_timer8   r8   r9   r   j  s:    

z+HyperlinkAvailabilityCheckWorker.limit_rate)N)r2   r3   r4   r]   r"   r    r   r   r5   rB   rh   rQ   r  r   r   r=   r   r^   r8   r8   rT   r9   r   y  s      Or   c                   @   s$   e Zd ZdZdZeddddZdS )HyperlinkCollector)ri      N)r   rF   c           
      K   s   t t| jj}|j}| jtjD ]T}d|vr0q"|d }| j	d|}|rN|}t
|}t|| jj|}||vr"|||< q"| jtjD ]\}	|	d d}|rd|v r| j	d|}|r|}t
|	}t|| jj|}||vr|||< qd S )NZrefurilinkcheck-process-urir   ?z://)r   rh   r   r   rk   Zdocumentfindallr   Z	referenceZemit_firstresultr-   r.   r   r0   Zimager   )
rS   r   r   rk   Zrefnoder/   Znewurir1   Zuri_infoZimgnoder8   r8   r9   r    s.    
zHyperlinkCollector.run)r2   r3   r4   ZbuildersZdefault_priorityr
   r  r8   r8   r8   r9   r    s   r  )r   r/   rF   c                 C   sH   t |}|jdkrD|jrD|jd}|sDd|j }t|j|dS dS )zRewrite anchor name of the hyperlink to github.com

    The hyperlink anchors in github.com are dynamically generated.  This rewrites
    them before checking and makes them comparable.
    z
github.comzuser-content-)fragmentN)r   hostnamer  r   r   _replace)r   r/   parsedZprefixedr  r8   r8   r9   rewrite_github_anchor  s    r  )r   r   rF   c                 C   s   t | jj D ]\}}zlzt|| jjt|< W n< tjyr } z"tt	d|j
|j W Y d}~n
d}~0 0 W | jj| q| jj| 0 qdS )zFCompile patterns in linkcheck_allowed_redirects to the regexp objects.z=Failed to compile regex in linkcheck_allowed_redirects: %r %sN)listr   r   r   rx   ry   errorr   r   r#   r   msgpop)r   r   r   r   excr8   r8   r9   #compile_linkcheck_allowed_redirects  s    
r  )r   rF   c                 C   s   |  t | t | dg d  | dg d  | di d  | dg d  | di d  | ddd  | dd d tg | d	d
d  | ddd  | ddgd  | ddd  | d | jdtdd ddddS )Nr   r   r   r   r   r   rD   r   r      r   Tr   z^!r  g     r@r
  zconfig-initedr	  )priorityZbuiltin)versionZparallel_read_safeZparallel_write_safe)	Zadd_builderrh   Zadd_post_transformr  Zadd_config_valuer7   Z	add_eventZconnectr  )r   r8   r8   r9   setup  s&    


r  )br]   r   rx   rp   r  rJ   r   r   Zemail.utilsr   Zhtml.parserr   osr   Zqueuer   r   	threadingr	   typingr
   r   r   r   r   r   r   r   r   r   r   urllib.parser   r   r   Zdocutilsr   Zdocutils.nodesr   r'   r   Zrequests.exceptionsr   r   r   Zsphinx.applicationr   Zsphinx.builders.dummyr   Zsphinx.configr    Zsphinx.deprecationr!   Zsphinx.environmentr"   Zsphinx.localer#   Z!sphinx.transforms.post_transformsr$   Zsphinx.utilr%   r&   Zsphinx.util.consoler(   r)   r*   r+   r,   Zsphinx.util.nodesr-   Z	getLoggerr2   r   ry   r   r.   r:   r>   rB   r=   r5   r7   ZCheckRequestTyper   r   r  r  rL   rM   r   rg   rh   r   r   r  r  r  r  r8   r8   r8   r9   <module>   sh   
4

	
 B9  $