+
    }i$                       ^ RI Ht ^ RIt^ RIHt ^ RIHt ^ RIHtH	t	H
t
 ^ RIHt ^ RIHtHtHtHtHtHt ^ RIHt ^ R	IHt ^ R
IHt ^ RIHt ^ RIHt ^ RIHt ^ RIH t H!t!H"t" ]'       d5   ^ RI#H$t$H%t% ^ RI&H't' ^ RI(H)t) ^ RI*H+t+ ^ RIH,t,H-t- ^ RI.H/t/ ^ RI0H1t1 ^ RI2H3t3 ]! ]44      t5 ! R R]
4      t6 ! R R4      t7]! R4       ! R R] 4      4       t8R# )    )annotationsN)asynccontextmanager)	getLogger)TYPE_CHECKINGAny	TypedDict)LRUCache)AsyncClientBrowser	HTTPErrorResponseTimeoutExceptionTransportError)
ProxyError)override)HttpHeaders)ROTATE_PROXY_ERRORS)
docs_group)
HttpClientHttpCrawlingResultHttpResponse)AsyncGeneratorAsyncIterator)	timedelta)	CookieJar)Request)
HttpMethodHttpPayload)	ProxyInfo)Session)
Statisticsc                  0    ] tR t^!t$ RtR]R&   R]R&   RtR# )_ClientCacheEntryz)Type definition for client cache entries.r
   clientCookieJar | None
cookie_jar N)__name__
__module____qualname____firstlineno____doc____annotations____static_attributes__r'       m/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/crawlee/http_clients/_impit.pyr#   r#   !   s    3  r/   r#   c                      ] tR t^(tRtR R lt]R R l4       t]R R l4       t]R R	 l4       t	R
 R lt
R R ltRtR# )_ImpitResponsezMAdapter class for `impit.Response` to conform to the `HttpResponse` protocol.c                    V ^8  d   QhRRRR/# )   responser   returnNoner'   )formats   "r0   __annotate___ImpitResponse.__annotate__+   s     " " "d "r/   c                	    Wn         R # N	_response)selfr5   s   &&r0   __init___ImpitResponse.__init__+   s    !r/   c                   V ^8  d   QhRR/# )r4   r6   strr'   )r8   s   "r0   r9   r:   /   s     0 0c 0r/   c                	@    \        V P                  P                  4      # r<   )rC   r>   http_versionr?   s   &r0   rE   _ImpitResponse.http_version.   s    4>>..//r/   c                   V ^8  d   QhRR/# )r4   r6   intr'   )r8   s   "r0   r9   r:   3   s     / /S /r/   c                	@    \        V P                  P                  4      # r<   )rI   r>   status_coderF   s   &r0   rK   _ImpitResponse.status_code2   s    4>>--..r/   c                   V ^8  d   QhRR/# )r4   r6   r   r'   )r8   s   "r0   r9   r:   7   s     9 9 9r/   c                	R    \        \        V P                  P                  4      4      # r<   )r   dictr>   headersrF   s   &r0   rP   _ImpitResponse.headers6   s    4 6 6788r/   c                   V ^8  d   QhRR/# )r4   r6   bytesr'   )r8   s   "r0   r9   r:   :   s     & &E &r/   c                	   "   V P                   P                  '       g   \        R 4      hV P                   P                  # 5i)zTUse `read_stream` to read the body of the Response received from the `stream` method)r>   	is_closedRuntimeErrorcontentrF   s   &r0   read_ImpitResponse.read:   s1     ~~'''uvv~~%%%s   >A c                   V ^8  d   QhRR/# )r4   r6   zAsyncIterator[bytes]r'   )r8   s   "r0   r9   r:   ?   s      #7 r/   c               	   "   V P                   P                  '       d   \        R 4      hV P                   P                  4         Rj  xL
  pV5x  K   LDR# 5i)zStream is already consumed.N)r>   is_stream_consumedrV   aiter_bytes)r?   chunks   & r0   read_stream_ImpitResponse.read_stream?   sJ     >>,,,<==#~~99;  e;s*   AAAA	A	AAAr=   N)r(   r)   r*   r+   r,   r@   propertyrE   rK   rP   rX   r_   r.   r'   r/   r0   r2   r2   (   sU    W" 0 0 / / 9 9&
 r/   r2   zHTTP clientsc                    a  ] tR t^GtRtRRRRRRRR/R	 V 3R
 lllt]RRRRRRRR/R R ll4       t]RRRRRRRRRRRR/R R ll4       t]	]RRRRRRRRRRRR/R R ll4       4       t
R R lt]R R l4       t]R R l4       tR tV ;t# )!ImpitHttpClienta   HTTP client based on the `impit` library.

This client uses the `impit` library to perform HTTP requests in crawlers (`BasicCrawler` subclasses)
and to manage sessions, proxies, and error handling.

See the `HttpClient` class for more common information about HTTP clients.

### Usage

```python
from crawlee.crawlers import HttpCrawler  # or any other HTTP client-based crawler
from crawlee.http_clients import ImpitHttpClient

http_client = ImpitHttpClient()
crawler = HttpCrawler(http_client=http_client)
```
persist_cookies_per_sessionThttp3Fverifybrowserfirefoxc               0    V ^8  d   QhRRRRRRRRRRR	R
/# )r4   rd   boolre   rf   rg   zBrowser | Noneasync_client_kwargsr   r6   r7   r'   )r8   s   "r0   r9   ImpitHttpClient.__annotate__[   sP     X X &*X 	X
 X  X  #X 
Xr/   c                  < \         SV `  VR7       W n        W0n        W@n        WPn        \        \        R,          \        3,          ! ^
R7      V n	        R# )a^  Initialize a new instance.

Args:
    persist_cookies_per_session: Whether to persist cookies per HTTP session.
    http3: Whether to enable HTTP/3 support.
    verify: SSL certificates used to verify the identity of requested hosts.
    browser: Browser to impersonate.
    async_client_kwargs: Additional keyword arguments for `impit.AsyncClient`.
)rd   N)maxsize)
superr@   _http3_verify_browser_async_client_kwargsr	   rC   r#   _client_by_proxy_url)r?   rd   re   rf   rg   rk   	__class__s   &$$$$,r0   r@   ImpitHttpClient.__init__[   sN    $ 	(C 	 	
 $7!$,S4Z9J-J$KTV$W!r/   sessionN
proxy_info
statisticstimeoutc               0    V ^8  d   QhRRRRRRRRR	R
RR/# )r4   requestr   rw   Session | Nonerx   ProxyInfo | Nonery   zStatistics | Nonerz   timedelta | Noner6   r   r'   )r8   s   "r0   r9   rl   y   sN     J JJ  	J
 %J &J "J 
Jr/   c               	  "   T P                  V'       d   VP                  MR V'       d   VP                  P                  MR 4      p TP	                  VP                  VP
                  VP                  VP                  '       d   \        VP                  4      MR V'       d   VP                  4       MR R7      G R j  xL
 pT'       d   TP#                  TP$                  4       \'        TP                  4      Tn        \+        \-        T4      R7      #  LV  \         d   p\        P                  ThR p?i\        \        3 d%   pT P                  T4      '       d   \         Thh R p?ii ; i5i)N)urlmethodrW   rP   rz   )http_response)_get_clientr   cookiesjarr|   r   payloadrP   rO   total_secondsr   asyncioTimeoutErrorr   r   _is_proxy_errorr   register_status_coderK   rC   
loaded_urlr   r2   )	r?   r|   rw   rx   ry   rz   r$   r5   excs	   &&$$$$   r0   crawlImpitHttpClient.crawlx   s     !!J*..Dah'//J]J]nrs	#^^KK~~18W__-d3:--/ ,  H ++H,@,@A .!x0HII'   	0&&C/	* 	##C(( c)	se   (E.E.=D D $D >D?D E.A
E.D E+"D33E+E+E&&E++E.r   GETrP   r   c               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r4   r   rC   r   r   rP   #HttpHeaders | dict[str, str] | Noner   HttpPayload | Nonerw   r}   rx   r~   rz   r   r6   r   r'   )r8   s   "r0   r9   rl      sZ     ( (( 	(
 5( $(  ( %( "( 
(r/   c               	L  "   \        V\        4      '       g   Vf   \        T;'       g    / 4      pT P                  V'       d   VP                  MR V'       d   VP
                  P                  MR 4      p TP                  TTTV'       d   \        V4      MR V'       d   VP                  4       MR R7      G R j  xL
 p	\!        T	4      #  L  \         d   p
\        P                  T
hR p
?
i\        \        3 d%   p
T P                  T
4      '       d   \        T
hh R p
?
ii ; i5i)N)r   r   rW   rP   rz   )
isinstancerO   r   r   r   r   r   r|   r   r   r   r   r   r   r   r   r2   r?   r   r   rP   r   rw   rx   rz   r$   r5   r   s   &&$$$$$$   r0   send_requestImpitHttpClient.send_request   s      gt$$!'--R0G!!J*..Dah'//J]J]nrs	#^^)0Wd3:--/ ,  H h''   	0&&C/	* 	##C(( c)	sj   (D$D$D$D$7C C !C ;C<C  D$C D!C))D!<D!=DD!!D$c               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r4   r   rC   r   r   rP   r   r   r   rw   r}   rx   r~   rz   r   r6   zAsyncGenerator[HttpResponse]r'   )r8   s   "r0   r9   rl      sZ     ! !! 	!
 5! $!  ! %! "! 
&!r/   c          	    	^  "   T P                  V'       d   VP                  MR V'       d   VP                  P                  MR 4      p TP	                  TTTV'       d   \        V4      MR V'       d   VP                  4       MR RR7      G R j  xL
 p	 \        T	4      5x  \        P                  ! R4      G R j  xL
  T	P                  4        R #  LC  \         d   p
\        P                  T
hR p
?
ii ; i L<  \        P                  ! R4      G R j  xL 
  T	P                  4        i ; i5i)NT)r   r   rW   rP   rz   streamg{Gz?)r   r   r   r   r|   rO   r   r   r   r   r2   sleepcloser   s   &&$$$$$$   r0   r   ImpitHttpClient.stream   s      !!J*..Dah'//J]J]nrs
	0#^^)0Wd3:--/ ,  H	 ** --%%%NN'   	0&&C/	0 &'--%%%NNsv   (D-D-C C 2C CC C9 !D-:C7;D-C C4C//C44D-9D*DD**D-c               $    V ^8  d   QhRRRRRR/# )r4   	proxy_urlz
str | Noner&   r%   r6   r
   r'   )r8   s   "r0   r9   rl      s#      Z =M R] r/   c           
     X   V P                   P                  V4      pV'       d   VR,          pVR,          pWRJ d   V# RVRV P                  RV P                  RRRV P                  /pVP                  V P                  4       \        R
/ VBRV/B p\        WBR	7      V P                   V&   V# )zRetrieve or create an HTTP client for the given proxy URL.

If a client for the specified proxy URL does not exist, create and store a new one.
r$   r&   proxyre   rf   follow_redirectsTrg   )r$   r&   r'   )	rt   getrp   rq   rr   updaters   r
   r#   )r?   r   r&   cached_datar$   client_cookie_jarkwargss   &&&    r0   r   ImpitHttpClient._get_client   s    
 //33I> *F +L 9 . YT[[dllt}}"
 	d//0=v=*=/@/f!!),r/   c                    V ^8  d   QhRRRR/# )r4   errorr   r6   rj   r'   )r8   s   "r0   r9   rl     s      y T r/   c                   a  \        S \        4      '       d   R# \        ;QJ d#    V 3R l\         4       F  '       g   K   RM	  RM! V 3R l\         4       4      '       d   R# R# )zDetermine whether the given error is related to a proxy issue.

Check if the error message contains known proxy-related error keywords.
Tc              3  >   <"   T F  q\        S4      9   x  K  	  R # 5ir<   )rC   ).0needler   s   & r0   	<genexpr>2ImpitHttpClient._is_proxy_error.<locals>.<genexpr>	  s     F2EU#2Es   F)r   ImpitProxyErroranyr   )r   s   fr0   r   ImpitHttpClient._is_proxy_error   s>     e_--3F2EF333F2EFFFr/   c                   V ^8  d   QhRR/# )r4   r6   r7   r'   )r8   s   "r0   r9   rl     s     * *t *r/   c                B   "   V P                   P                  4        R# 5i)z+Clean up resources used by the HTTP client.N)rt   clearrF   s   &r0   cleanupImpitHttpClient.cleanup  s      	!!'')s   )rs   rr   rt   rp   rq   )r(   r)   r*   r+   r,   r@   r   r   r   r   r   r   staticmethodr   r   r.   __classcell__)ru   s   @r0   rc   rc   G   sZ   $X -1X 	X
 X #,X X: J #'	J
 (,J )-J %)J JB ( #	(
 8<( '+( #'( (,( %)( (B ! #	!
 8<! '+! #'! (,! %)!  !F>   * *r/   rc   )9
__future__r   r   
contextlibr   loggingr   typingr   r   r   
cachetoolsr	   impitr
   r   r   r   r   r   r   r   typing_extensionsr   crawlee._typesr   crawlee._utils.blockedr   crawlee._utils.docsr   crawlee.errorscrawlee.http_clientsr   r   r   collections.abcr   r   datetimer   http.cookiejarr   crawleer   r   r   crawlee.proxy_configurationr   crawlee.sessionsr    crawlee.statisticsr!   r(   loggerr#   r2   rc   r'   r/   r0   <module>r      s    "  *  0 0  ] ] / & & 6 * % M M="(65(-	8	!	 ! > NI*j I* I*r/   