+
    }i2                    
   ^ RI Ht ^ RIt^ RIHt ^ RIHt ^ RIHtH	t	H
t
 ^ RIHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RI H!t!H"t"H#t# ^ RI$H%t% ^ RI&H't' ^ RI(Ht ^ RI)H*t*H+t+H,t, ]'       dC   ^ RI-H.t. ^ RI/H0t0 ^ RIH1t1 ^ RIH2t3 ^ RIH4t4 ^ RI5H"t6 ^ RI7H2t2 ^ RI H"t" ^ RI8H9t9 ^ RI:H;t; ^ RI<H=t=  ! R R]4      t> ! R R ]4      t? ! R! R"4      t@]'! R#4       ! R$ R%]*4      4       tAR# )&    )annotationsN)asynccontextmanager)Cookie)TYPE_CHECKINGAnycast)CurlInfo)CurlHttpVersion)AsyncSession)Cookies)
CurlMorsel)
ProxyError)RequestException)Timeout)DEFAULT_CHROME)override)HttpHeaders
HttpMethodHttpPayload)ROTATE_PROXY_ERRORS)
docs_group)
HttpClientHttpCrawlingResultHttpResponse)AsyncGenerator)	timedelta)Curl)Request)Response)r   )	ProxyInfo)Session)
Statisticsc                  B    ] tR t^)t]R R l4       t]R R l4       tRtR# )_EmptyCookiesc                    V ^8  d   QhRRRR/# )   requestCurlRequestreturnlist[CurlMorsel] )formats   "x/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/crawlee/http_clients/_curl_impersonate.py__annotate___EmptyCookies.__annotate__+   s      K <L     c                	    . # Nr+   )selfr'   s   &&r-   get_cookies_for_curl"_EmptyCookies.get_cookies_for_curl*   s    	r0   c                    V ^8  d   QhRRRR/# )r&   morselsr*   r)   Noner+   )r,   s   "r-   r.   r/   /   s      0@ T r0   c                	    R # r2   r+   )r3   r7   s   &&r-   update_cookies_from_curl&_EmptyCookies.update_cookies_from_curl.   s    r0   r+   N)__name__
__module____qualname____firstlineno__r   r4   r:   __static_attributes__r+   r0   r-   r$   r$   )   s(       r0   r$   c                  8   a  ] tR t^3t]R V 3R ll4       tRtV ;t# )_AsyncSessionc               $    V ^8  d   QhRRRRRR/# )r&   argsr   kwargsr)   r8   r+   )r,   s   "r-   r.   _AsyncSession.__annotate__5   s!     ( (c (S (T (r0   c                	F   < \         SV `  ! V/ VB  \        4       V n        R # r2   )super__init__r$   _cookies)r3   rD   rE   	__class__s   &*,r-   rI   _AsyncSession.__init__4   s    $)&)%r0   )rJ   )r<   r=   r>   r?   r   rI   r@   __classcell__rK   s   @r-   rB   rB   3   s    ( (r0   rB   c                      ] tR t^:tRtR R lt]R R l4       t]R R l4       t]R R	 l4       t	R
 R lt
R R ltRtR# )_CurlImpersonateResponsezZAdapter class for `curl_cffi.requests.Response` to conform to the `HttpResponse` protocol.c                    V ^8  d   QhRRRR/# )r&   responser   r)   r8   r+   )r,   s   "r-   r.   %_CurlImpersonateResponse.__annotate__=   s     " " "d "r0   c                	    Wn         R # r2   	_response)r3   rR   s   &&r-   rI   !_CurlImpersonateResponse.__init__=   s    !r0   c                   V ^8  d   QhRR/# )r&   r)   strr+   )r,   s   "r-   r.   rS   A   s     Q Qc Qr0   c                	2   V P                   P                  \        P                  8X  d   R # V P                   P                  \        P                  8X  d   R# V P                   P                  \        P
                  8X  d   R# V P                   P                  \        P                  \        P                  \        P                  09   d   R# V P                   P                  \        P                  8X  d   R# \        RV P                   P                   24      h)NONEzHTTP/1.0zHTTP/1.1zHTTP/2zHTTP/3zUnknown HTTP version: )rV   http_versionr
   r[   V1_0V1_1V2_0V2TLSV2_PRIOR_KNOWLEDGEV3
ValueErrorr3   s   &r-   r\   %_CurlImpersonateResponse.http_version@   s    >>&&/*>*>>>>&&/*>*>>>>&&/*>*>>>>&&  !!..+
 

 >>&&/*<*<<1$..2M2M1NOPPr0   c                   V ^8  d   QhRR/# )r&   r)   intr+   )r,   s   "r-   r.   rS   T   s     * *S *r0   c                	.    V P                   P                  # r2   )rV   status_coderd   s   &r-   ri   $_CurlImpersonateResponse.status_codeS   s    ~~)))r0   c                   V ^8  d   QhRR/# )r&   r)   r   r+   )r,   s   "r-   r.   rS   X   s     c c cr0   c                	    \        V P                  P                  P                  4        UUu/ uF  w  rV'       g   K  WbK  	  upp4      # u uppi r2   )r   rV   headersitems)r3   keyvalues   &  r-   rm    _CurlImpersonateResponse.headersW   s=    9O9O9U9U9Wa9W:3[`JCJ9Wabbas
   A
A
c                   V ^8  d   QhRR/# )r&   r)   bytesr+   )r,   s   "r-   r.   rS   [   s     & &E &r0   c                	   "   V P                   P                  '       d   \        R 4      hV P                   P                  # 5i)zTUse `read_stream` to read the body of the Response received from the `stream` method)rV   astream_taskRuntimeErrorcontentrd   s   &r-   read_CurlImpersonateResponse.read[   s1     >>&&&uvv~~%%%s   >A c                   V ^8  d   QhRR/# )r&   r)   zAsyncGenerator[bytes, None]r+   )r,   s   "r-   r.   rS   a   s      #> r0   c               	  "   V P                   P                  '       g   \        R 4      h\        V P                   P                  \        P
                  4      '       d6   V P                   P                  P                  4       '       d   \        R4      hV P                   P                  4         Rj  xL
  pV5x  K   LDR# 5i)z?Cannot read stream, Response not obtained from `stream` method.z,Cannot read stream, it was already consumed.N)rV   ru   rv   
isinstanceasyncioFuturedoneaiter_content)r3   chunks   & r-   read_stream$_CurlImpersonateResponse.read_streama   s     ~~***`aadnn117>>BBt~~GbGbGgGgGiGiMNN>>779 	 	%K	9s0   BC%C-C 1B>2C 5	C>C  CrU   N)r<   r=   r>   r?   __doc__rI   propertyr\   ri   rm   rx   r   r@   r+   r0   r-   rP   rP   :   sY    d" Q Q$ * * c c& r0   rP   zHTTP clientsc                  "  a  ] tR t^ltRtRR/R V 3R lllt]RRRRR	RR
R/R R ll4       t]RRRRRRRRRRR
R/R R ll4       t]	]RRRRRRRRRRR
R/R R ll4       4       t
R R ltR R lt]R R l4       t]R R l4       tR R ltRtV ;t# ) CurlImpersonateHttpClienta  HTTP client based on the `curl-cffi` library.

This client uses the `curl-cffi` library to perform HTTP requests in crawlers (`BasicCrawler` subclasses)
and to manage sessions, proxies, and error handling.

See the `HttpClient` class for more common information about HTTP clients.

### Usage

```python
from crawlee.crawlers import HttpCrawler  # or any other HTTP client-based crawler
from crawlee.http_clients import CurlImpersonateHttpClient

http_client = CurlImpersonateHttpClient()
crawler = HttpCrawler(http_client=http_client)
```
persist_cookies_per_sessionTc               $    V ^8  d   QhRRRRRR/# )r&   r   boolasync_session_kwargsr   r)   r8   r+   )r,   s   "r-   r.   &CurlImpersonateHttpClient.__annotate__   s/     E E &*E !$	E
 
Er0   c                  < \         SV `  VR7       W n        \        \        R,          \
        3,          ! 4       V n        R# )zInitialize a new instance.

Args:
    persist_cookies_per_session: Whether to persist cookies per HTTP session.
    async_session_kwargs: Additional keyword arguments for `curl_cffi.requests.AsyncSession`.
)r   N)rH   rI   _async_session_kwargsdictrY   r   _client_by_proxy_url)r3   r   r   rK   s   &$,r-   rI   "CurlImpersonateHttpClient.__init__   s;     	(C 	 	
 &:"$(t\)A$B$D!r0   sessionN
proxy_info
statisticstimeoutc               0    V ^8  d   QhRRRRRRRRR	R
RR/# )r&   r'   r   r   Session | Noner   ProxyInfo | Noner   zStatistics | Noner   timedelta | Noner)   r   r+   )r,   s   "r-   r.   r      sF     &
 &
&
  	&

 %&
 &&
 "&
 
&
r0   c          	     	j  "   T P                  V'       d   VP                  MR 4      p TP                  VP                  V P                  VP                  4      VP
                  VP                  V'       d   VP                  P                  MR V'       d   VP                  4       MR R7      G R j  xL
 pT'       d   TP!                  TP"                  4       T P$                  '       dQ   T'       dI   TP&                  '       d7   T P)                  TP&                  4      p	TP                  P+                  T	4       TP                  Tn        \/        \1        T4      R7      #  L  \         d   p\        P                  ThR p?i\         d%   pT P                  T4      '       d   \        Thh R p?ii ; i5i)Nurlmethodrm   datacookiesr   )http_response)_get_clientr   r'   _convert_methodr   rm   payloadr   jartotal_secondsr   r}   TimeoutErrorCurlRequestError_is_proxy_errorr   register_status_coderi   _persist_cookies_per_sessioncurl_get_cookiesstore_cookies
loaded_urlr   rP   )
r3   r'   r   r   r   r   clientrR   excresponse_cookiess
   &&$$$$    r-   crawlCurlImpersonateHttpClient.crawl   sE     !!J*..DI	#^^KK++GNN;__/6++D3:--/ ,  H ++H,@,@A,,,X]]]#00?OO))*:;%\\!28<
 	
1  	0&&C/ 	##C(( c)	sr   &F3AE" 7E" E" 0E 1E" 5F3=,F3*F32F3AF3 E" "F0-E>>F0F0F++F00F3r   GETrm   r   c               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r&   r   rY   r   r   rm   #HttpHeaders | dict[str, str] | Noner   HttpPayload | Noner   r   r   r   r   r   r)   r   r+   )r,   s   "r-   r.   r      sZ     %2 %2%2 	%2
 5%2 $%2  %2 %%2 "%2 
%2r0   c          	     	&  "   \        V\        4      '       g   Vf   \        T;'       g    / 4      pV'       d   VP                  MR pV P	                  V4      p	 T	P                  TV P                  V4      V'       d   \        V4      MR TV'       d   VP                  P                  MR V'       d   VP                  4       MR R7      G R j  xL
 p
T P                   '       dQ   T'       dI   T
P"                  '       d7   T P%                  T
P"                  4      pTP                  P'                  T4       \)        T
4      #  Lq  \         d   p\        P                  ThR p?i\         d%   pT P                  T4      '       d   \        Thh R p?ii ; i5i)Nr   )r|   r   r   r   r   r'   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   rP   r3   r   r   rm   r   r   r   r   	proxy_urlr   rR   r   r   s   &&$$$$$$     r-   send_request&CurlImpersonateHttpClient.send_request   s6     gt$$!'--R0G&0JNNd	!!),	#^^++F3)0Wd/6++D3:--/ ,  H ,,,X]]]#00?OO))*:;'11'  	0&&C/ 	##C(( c)	s|   (FFF#E  >E  E  2E  D>E  F#F+F=AF>E   FEF)F*F		FFc               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r&   r   rY   r   r   rm   r   r   r   r   r   r   r   r   r   r)   zAsyncGenerator[HttpResponse]r+   )r,   s   "r-   r.   r      sZ     )$ )$)$ 	)$
 5)$ $)$  )$ %)$ ")$ 
&)$r0   c          
    	  "   \        V\        4      '       g   Vf   \        T;'       g    / 4      pV'       d   VP                  MR pV P	                  V4      p	 T	P                  TV P                  V4      V'       d   \        V4      MR TV'       d   VP                  P                  MR RV'       d   VP                  4       MR R7      G R j  xL
 p
T P                   '       dQ   T'       dI   T
P"                  '       d7   T P%                  T
P"                  4      pTP                  P'                  T4        \)        T
4      5x  T
P+                  4       G R j  xL
  R #  L  \         d   p\        P                  ThR p?i\         d%   pT P                  T4      '       d   \        Thh R p?ii ; i LY  T
P+                  4       G R j  xL 
  i ; i5i)NT)r   r   rm   r   r   streamr   )r|   r   r   r   r   r'   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   rP   acloser   s   &&$$$$$$     r-   r    CurlImpersonateHttpClient.stream   sd     gt$$!'--R0G&0JNNd	!!),	#^^++F3)0Wd/6++D3:--/ ,  H  ,,,X]]]#00?OO))*:;	$*844//###/  	0&&C/ 	##C(( c)	 $(//###s   (GGG#E >E E 3E EE G$G,G>6G5F2 GF0GE F-*E;;F-F-	F((F--G2GG	GGc                    V ^8  d   QhRRRR/# )r&   r   z
str | Noner)   r   r+   )r,   s   "r-   r.   r     s     4 4Z 4L 4r0   c                    WP                   9  d>   RVR\        /pVP                  V P                  4       \	        R/ VB V P                   V&   V P                   V,          # )a!  Retrieve or create an asynchronous HTTP session for the given proxy URL.

Check if an `AsyncSession` already exists for the specified proxy URL. If no session is found,
create a new one with the provided proxy settings and additional session options.
Store the new session for future use.
proxyimpersonater+   )r   CURL_DEFAULT_CHROMEupdater   rB   )r3   r   rE   s   && r-   r   %CurlImpersonateHttpClient._get_client  sd     555 2&F MM$445 4A3J63JD%%i0((33r0   c                    V ^8  d   QhRRRR/# )r&   r   r   r)   CurlHttpMethodr+   )r,   s   "r-   r.   r   )  s      i  ij  i^  ir0   c                   VP                  4       pT;R8X  d    R# ;R8X  d    R# ;R8X  d    R# ;R8X  d    R# ;R8X  d    R# ;R8X  d    R# ;R8X  d    R# R8X  d   R#  \        R	V R
V P                  P                   R24      h)zConvert from Crawlee HTTP method to curl-cffi HTTP method.

Args:
    method: Crawlee HTTP method.

Returns:
    Corresponding curl-cffi HTTP method.

Raises:
    ValueError: If the provided HTTP method is not supported.
r   POSTPUTDELETEOPTIONSHEADTRACEPATCHzHTTP method z is not supported in .)upperrc   rK   r<   )r3   r   method_uppers   && r-   r   )CurlImpersonateHttpClient._convert_method)  sl     ||~  <x7LT^^MdMdLeef!ghhr0   c                    V ^8  d   QhRRRR/# )r&   errorr   r)   r   r+   )r,   s   "r-   r.   r   L  s      / D r0   c                   a  \         ;QJ d#    V 3R l\         4       F  '       g   K   RM	  RM! V 3R l\         4       4      '       d   R# \        S \        4      '       d   R# R# )zDetermine whether the given error is related to a proxy issue.

Check if the error message contains known proxy-related error keywords or if it is an instance
of `CurlProxyError`.
c              3  >   <"   T F  q\        S4      9   x  K  	  R # 5ir2   )rY   ).0needler   s   & r-   	<genexpr><CurlImpersonateHttpClient._is_proxy_error.<locals>.<genexpr>R  s     F2EU#2Es   TF)anyr   r|   CurlProxyError)r   s   fr-   r   )CurlImpersonateHttpClient._is_proxy_errorK  s>     3F2EF333F2EFFFe^,,r0   c                    V ^8  d   QhRRRR/# )r&   r   r   r)   zlist[Cookie]r+   )r,   s   "r-   r.   r   [  s      4 L r0   c                	   \         \        ,          ! 4       p\        R V P                  \        P
                  4      4      pV F:  p\        P                  ! V4      pVP                  4       pVP                  V4       K<  	  V# )zlist[bytes])
listr   r   getinfor	   
COOKIELISTr   from_curl_formatto_cookiejar_cookieappend)r   r   cookie_listcurl_cookiecurl_morselcookies   &     r-   r   &CurlImpersonateHttpClient._get_cookiesZ  se    v,. =$,,x7J7J*KL&K$55kBK 446FNN6" '
 r0   c                   V ^8  d   QhRR/# )r&   r)   r8   r+   )r,   s   "r-   r.   r   h  s     * *t *r0   c                	   "   V P                   P                  4        F  pVP                  4       G R j  xL
  K  	  V P                   P                  4        R #  L$5ir2   )r   valuescloseclear)r3   r   s   & r-   cleanup!CurlImpersonateHttpClient.cleanuph  sD     //668F,,.   9!!'') !s   1AA%A)r   r   )r<   r=   r>   r?   r   rI   r   r   r   r   r   r   r   staticmethodr   r   r   r@   rM   rN   s   @r-   r   r   l   sD   $E -1E E& &
 #'	&

 (,&
 )-&
 %)&
 &
P %2 #	%2
 8<%2 '+%2 #'%2 (,%2 %)%2 %2N )$ #	)$
 8<)$ '+)$ #')$ (,)$ %))$  )$V40 iD    * *r0   r   )B
__future__r   r}   
contextlibr   http.cookiejarr   typingr   r   r   	curl_cffir	   curl_cffi.constr
   curl_cffi.requestsr   curl_cffi.requests.cookiesr   CurlCookiesr   curl_cffi.requests.exceptionsr   r   r   r   r   curl_cffi.requests.impersonater   r   typing_extensionsr   crawlee._typesr   r   r   crawlee._utils.blockedr   crawlee._utils.docsr   crawlee.errorscrawlee.http_clientsr   r   r   collections.abcr   datetimer   r   r   r(   r   curl_cffi.requests.sessionr   crawleecrawlee.proxy_configurationr    crawlee.sessionsr!   crawlee.statisticsr"   r$   rB   rP   r   r+   r0   r-   <module>r     s    "  * ! + +  + + = 1 F N 1 P & ? ? 6 * % M M."9+G)5(-K (L (/ /d N~*
 ~* ~*r0   