B ufx@@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlmZddlmZddl m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6erHddlm7Z7ne8Z7e9e:Z;eeGddde?Z@eddddZAGddde?ZBeZOGd?d@d@eZPGdAdBdBZQdS)EzO The main purpose of this module is to expose LinkCollector.collect_sources(). N) HTMLParser)Values) TYPE_CHECKINGCallableDictIterableListMutableMapping NamedTupleOptionalSequenceTupleUnion)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_file)redact_auth_from_url)vcs)CandidatesFromPage LinkSource build_source)Protocol)urlreturncCs6x0tjD]&}||r|t|dkr|SqWdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rZschemeslower startswithlen)r schemer&_/home/ankuromar296_gmail_com/myenv/lib/python3.7/site-packages/pip/_internal/index/collector.py_match_vcs_scheme7s r(cs&eZdZeeddfdd ZZS)_NotAPIContentN) content_type request_descr!cst||||_||_dS)N)super__init__r*r+)selfr*r+) __class__r&r'r-Csz_NotAPIContent.__init__)__name__ __module__ __qualname__strr- __classcell__r&r&)r/r'r)Bsr))responser!cCs6|jdd}|}|dr$dSt||jjdS)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. z Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+htmlz#application/vnd.pypi.simple.v1+jsonN)headersgetr"r#r)requestmethod)r5r*content_type_lr&r&r'_ensure_api_headerIs r<c@s eZdZdS)_NotHTTPN)r0r1r2r&r&r&r'r=_sr=)r sessionr!cCsFtj|\}}}}}|dkr$t|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitr=headrr<)r r>r%netlocpathqueryfragmentrespr&r&r'_ensure_api_responsecs rKcCsztt|jrt||dtdt||j|ddddgddd }t |t |td t||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. )r>zGetting page %sz, z#application/vnd.pypi.simple.v1+jsonz*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r7zFetched page %s as %sz Content-Typer6) rrfilenamerKloggerdebugrr8joinrr<r7)r r>rJr&r&r'_get_simple_responseus"   rQ)r7r!cCs<|r8d|kr8tj}|d|d<|d}|r8t|SdS)z=Determine if we have any encoding information in our headers.z Content-Typez content-typecharsetN)emailmessageMessage get_paramr3)r7mrRr&r&r'_get_encoding_from_headerss    rXc@s:eZdZdddddZeedddZed d d ZdS) CacheablePageContent IndexContentN)pager!cCs|js t||_dS)N)cache_link_parsingAssertionErrorr[)r.r[r&r&r'r-s zCacheablePageContent.__init__)otherr!cCst|t|o|jj|jjkS)N) isinstancetyper[r )r.r^r&r&r'__eq__szCacheablePageContent.__eq__)r!cCs t|jjS)N)hashr[r )r.r&r&r'__hash__szCacheablePageContent.__hash__) r0r1r2r-objectboolraintrcr&r&r&r'rYsrYc@s eZdZdeedddZdS) ParseLinksrZ)r[r!cCsdS)Nr&)r.r[r&r&r'__call__szParseLinks.__call__N)r0r1r2rrrhr&r&r&r'rgsrg)fnr!csLtjddtttdfdd tdttdfdd }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. N)maxsize)cacheable_pager!cst|jS)N)listr[)rk)rir&r'wrappersz*with_cached_index_content..wrapperrZ)r[r!cs|jrt|St|S)N)r\rYrl)r[)rirmr&r'wrapper_wrappers z2with_cached_index_content..wrapper_wrapper) functools lru_cacherYrrwraps)rirnr&)rirmr'with_cached_index_contents  rrrZ)r[r!c cs|j}|drXt|j}x2|dgD]"}t||j }|dkrJq.|Vq.WdSt |j }|j pjd}| |j ||j }|jp|}x.|jD]$} tj| ||d}|dkrq|VqWdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. z#application/vnd.pypi.simple.v1+jsonfilesNzutf-8)page_urlbase_url)r*r"r#jsonloadscontentr8r from_jsonr HTMLLinkParserencodingfeeddecoderuanchors from_element) r[r;datafilelinkparserr{r ruanchorr&r&r' parse_linkss&        rc@s<eZdZdZd eeeeeeddddZeddd Z dS) rZz5Represents one response (or page), along with its URLTN)rxr*r{r r\r!cCs"||_||_||_||_||_dS)am :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. N)rxr*r{r r\)r.rxr*r{r r\r&r&r'r-s zIndexContent.__init__)r!cCs t|jS)N)rr )r.r&r&r'__str__szIndexContent.__str__)T) r0r1r2__doc__bytesr3r rer-rr&r&r&r'rZscsneZdZdZeddfdd ZeeeeeefddddZ eeeeefeed d d Z Z S) rzzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. N)r r!cs$tjdd||_d|_g|_dS)NT)Zconvert_charrefs)r,r-r rur~)r.r )r/r&r'r-#szHTMLLinkParser.__init__)tagattrsr!cCsH|dkr,|jdkr,||}|dk rD||_n|dkrD|jt|dS)Nbasea)ruget_hrefr~appenddict)r.rrhrefr&r&r'handle_starttag*s  zHTMLLinkParser.handle_starttag)rr!cCs"x|D]\}}|dkr|SqWdS)Nrr&)r.rnamevaluer&r&r'r2szHTMLLinkParser.get_href) r0r1r2rr3r-rr r rrr4r&r&)r/r'rzs"rz).N)rreasonmethr!cCs|dkrtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rNrO)rrrr&r&r'_handle_get_simple_fail9srT)r5r\r!cCs&t|j}t|j|jd||j|dS)Nz Content-Type)r{r r\)rXr7rZrxr )r5r\r{r&r&r'_make_index_contentCs r)rr>r!c Cs|jddd}t|}|r0td||dStj|\}}}}}}|dkrtj tj |r| dsv|d7}tj|d}td|yt||d }WnDtk rtd |Yn2tk r}ztd ||j|jWdd}~XYntk r$}zt||Wdd}~XYntk rP}zt||Wdd}~XYntk r}z$d } | t|7} t|| tjd Wdd}~XYndtjk r}zt|d|Wdd}~XYn0tjk rt|dYnXt||jdSdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %s)r>z`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)r\) r splitr(rNwarningrBrCurlparseosrGisdirr9 url2pathnameendswithurljoinrOrQr=r)r+r*rrrrr3inforConnectionErrorTimeoutrr\) rr>r Z vcs_schemer%_rGrJexcrr&r&r'_get_index_contentPsP      rc@s.eZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r0r1r2r r r__annotations__r&r&r&r'rs rc@sxeZdZdZeeddddZedeee dddd Z e e e d d d Zeeed ddZe eedddZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. N)r> search_scoper!cCs||_||_dS)N)rr>)r.r>rr&r&r'r-szLinkCollector.__init__F)r>optionssuppress_no_indexr!cCsd|jg|j}|jr8|s8tdddd|Dg}|jp@g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdS)N)r).0r r&r&r' sz'LinkCollector.create..)rrno_index)r>r) index_urlextra_index_urlsrrNrOrPrrcreater)clsr>rrrrrlink_collectorr&r&r'rs    zLinkCollector.create)r!cCs|jjS)N)rr)r.r&r&r'rszLinkCollector.find_links)locationr!cCst||jdS)z> Fetch an HTML page containing package links. )r>)rr>)r.rr&r&r'fetch_responseszLinkCollector.fetch_response) project_namecandidates_from_pager!cstfddj|D}tfddjD}ttj rddt ||D}t |d|dg|}t d|tt|t|d S) Nc3s$|]}t|jjdddVqdS)F)rpage_validator expand_dirr\N)rr>is_secure_origin)rloc)rr.r&r'rsz0LinkCollector.collect_sources..c3s$|]}t|jjdddVqdS)T)rrrr\N)rr>r)rr)rr.r&r'rscSs*g|]"}|dk r|jdk rd|jqS)Nz* )r)rsr&r&r' sz1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrZget_index_urls_locationsvaluesrrN isEnabledForloggingDEBUG itertoolschainr$rOrPrrl)r.rrZindex_url_sourcesZfind_links_sourceslinesr&)rr.r'collect_sourcess   zLinkCollector.collect_sources)F)r0r1r2rrrr- classmethodrrerpropertyrr3rrr rZrrrrr&r&r&r'rsr)N)T)Rrr email.messagerSrorrvrr urllib.parserBurllib.requestZ html.parserroptparsertypingrrrrrr r r r r r pip._vendorrZpip._vendor.requestsrZpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkrZ!pip._internal.models.search_scoperpip._internal.network.sessionrZpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.vcsrsourcesrrrrrd getLoggerr0rNr3ZResponseHeadersr( Exceptionr)r<r=rKrQrXrYrgrrrrZrzrrerrrrr&r&r&r'sd  4             ?   =