+
    }i=0                       ^ RI Ht ^ RIt^ RIHt ^ RIHt ^ RIHtH	t	H
t
 ^ RIt^ RIHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IHt ^ RIHt ^ RIHtHtHt ]'       d5   ^ RIHtHt ^ RIH t  ^ RI!H"t" ^ RI#H$t$ ^ RIH%t%H&t& ^ RI'H(t( ^ RI)H*t* ^ RI+H,t, ]! ]-4      t. ! R R4      t/ ! R R]P`                  4      t1]! R4       ! R R]4      4       t2R# )    )annotationsN)asynccontextmanager)	getLogger)TYPE_CHECKINGAnycast)override)HttpHeaders)ROTATE_PROXY_ERRORS)
docs_group)
ProxyError)HeaderGenerator)
HttpClientHttpCrawlingResultHttpResponse)AsyncGeneratorAsyncIterator)	timedelta)
SSLContext)Request)
HttpMethodHttpPayload)	ProxyInfo)Session)
Statisticsc                      ] tR t^ tRtR R lt]R R l4       t]R R l4       t]R R	 l4       t	R
 R lt
R R ltRtR# )_HttpxResponsezMAdapter class for `httpx.Response` to conform to the `HttpResponse` protocol.c                    V ^8  d   QhRRRR/# )   responsehttpx.ResponsereturnNone )formats   "m/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/crawlee/http_clients/_httpx.py__annotate___HttpxResponse.__annotate__#   s     " " "D "    c                	    Wn         R # N	_response)selfr    s   &&r&   __init___HttpxResponse.__init__#   s    !r)   c                   V ^8  d   QhRR/# )r   r"   strr$   )r%   s   "r&   r'   r(   '   s     + +c +r)   c                	.    V P                   P                  # r+   )r-   http_versionr.   s   &r&   r4   _HttpxResponse.http_version&   s    ~~***r)   c                   V ^8  d   QhRR/# )r   r"   intr$   )r%   s   "r&   r'   r(   +   s     * *S *r)   c                	.    V P                   P                  # r+   )r-   status_coder5   s   &r&   r:   _HttpxResponse.status_code*   s    ~~)))r)   c                   V ^8  d   QhRR/# )r   r"   r
   r$   )r%   s   "r&   r'   r(   /   s     9 9 9r)   c                	R    \        \        V P                  P                  4      4      # r+   )r
   dictr-   headersr5   s   &r&   r?   _HttpxResponse.headers.   s    4 6 6788r)   c                   V ^8  d   QhRR/# )r   r"   bytesr$   )r%   s   "r&   r'   r(   2   s     , ,E ,r)   c                	   "   V P                   P                  '       g   \        R 4      hV P                   P                  4       G Rj  xL
 #  L5i)zTUse `read_stream` to read the body of the Response received from the `stream` methodN)r-   	is_closedRuntimeErrorareadr5   s   &r&   read_HttpxResponse.read2   s:     ~~'''uvv^^))++++s   AAAAc                   V ^8  d   QhRR/# )r   r"   zAsyncIterator[bytes]r$   )r%   s   "r&   r'   r(   7   s      #7 r)   c               	   "   V P                   P                  '       d   \        R 4      hV P                   P                  4         Rj  xL
  pV5x  K   LDR# 5i)zStream is already consumed.N)r-   is_stream_consumedrE   aiter_bytes)r.   chunks   & r&   read_stream_HttpxResponse.read_stream7   sJ     >>,,,<==#~~99;  e;s*   AAAA	A	AAAr,   N)__name__
__module____qualname____firstlineno____doc__r/   propertyr4   r:   r?   rG   rN   __static_attributes__r$   r)   r&   r   r       sU    W" + + * * 9 9,
 r)   r   c                  <   a  ] tR t^?tRt]R V 3R ll4       tRtV ;t# )_HttpxTransporta"  HTTP transport adapter that stores response cookies in a `Session`.

This transport adapter modifies the handling of HTTP requests to update the session cookies
based on the response cookies, ensuring that the cookies are stored in the session object
rather than the `HTTPX` client itself.
c                    V ^8  d   QhRRRR/# )r   requesthttpx.Requestr"   r!   r$   )r%   s   "r&   r'   _HttpxTransport.__annotate__H   s     
 
- 
N 
r)   c                	T  <"   \         SV `  V4      G R j  xL
 pWn        \        RVP                  P                  R4      4      ;p'       d9   VP                  P                  \        VP                  P                  4      4       RVP                  9   d   VP                  R V#  L5i)Nr   crawlee_sessionz
Set-Cookie)superhandle_async_requestrZ   r   
extensionsgetcookiesstore_cookieslistjarr?   )r.   rZ   r    session	__class__s   &&  r&   r`   $_HttpxTransport.handle_async_requestG   s     5g>>"9g&8&8&<&<=N&OPP7POO))$x/?/?/C/C*DE8+++  . ?s   B(B&BB(r$   )	rP   rQ   rR   rS   rT   r	   r`   rV   __classcell__rh   s   @r&   rX   rX   ?   s     
 
r)   rX   zHTTP clientsc                  :  a  ] tR t^UtRt]! 4       tRRRRRRRRR]/R V 3R	 lllt]R
RRRRRRR/R R ll4       t	]RRRRRRR
RRRRR/R R ll4       t
]]RRRRRRR
RRRRR/R R ll4       4       tR$R R lltR R ltR R lt]R R  l4       tR! R" ltR#tV ;t# )%HttpxHttpClienta   HTTP client based on the `HTTPX` library.

This client uses the `HTTPX` library to perform HTTP requests in crawlers (`BasicCrawler` subclasses)
and to manage sessions, proxies, and error handling.

See the `HttpClient` class for more common information about HTTP clients.

### Usage

```python
from crawlee.crawlers import HttpCrawler  # or any other HTTP client-based crawler
from crawlee.http_clients import HttpxHttpClient

http_client = HttpxHttpClient()
crawler = HttpCrawler(http_client=http_client)
```
persist_cookies_per_sessionThttp1http2verifyheader_generatorc               4    V ^8  d   QhRRRRRRRRRRR	R
RR/# )r   rn   boolro   rp   rq   zstr | bool | SSLContextrr   zHeaderGenerator | Noneasync_client_kwargsr   r"   r#   r$   )r%   s   "r&   r'   HttpxHttpClient.__annotate__k   s[     !J !J &*!J 	!J
 !J (!J 1!J  #!J 
!Jr)   c                 < \         SV `  VR7       W n        W0n        W`n        WPn        \        P                  ! VR7      V n        RV n	        \        \        R,          \        P                  3,          ! 4       V n        R# )a  Initialize a new instance.

Args:
    persist_cookies_per_session: Whether to persist cookies per HTTP session.
    http1: Whether to enable HTTP/1.1 support.
    http2: Whether to enable HTTP/2 support.
    verify: SSL certificates used to verify the identity of requested hosts.
    header_generator: Header generator instance to use for generating common headers.
    async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`.
)rn   )rq   N)r_   r/   _http1_http2_async_client_kwargs_header_generatorhttpxcreate_ssl_context_ssl_context
_transportr>   r2   AsyncClient_client_by_proxy_url)r.   rn   ro   rp   rq   rr   ru   rh   s   &$$$$$,r&   r/   HttpxHttpClient.__init__k   sk    ( 	(C 	 	
 $7!!1!44FC26$(tU5F5F)F$G$I!r)   rg   N
proxy_info
statisticstimeoutc               0    V ^8  d   QhRRRRRRRRR	R
RR/# )r   rZ   r   rg   Session | Noner   ProxyInfo | Noner   zStatistics | Noner   timedelta | Noner"   r   r$   )r%   s   "r&   r'   rv      sF     &
 &
&
  	&

 %&
 &&
 "&
 
&
r)   c          
     	H  "   T P                  V'       d   VP                  MR 4      pV P                  VP                  4      pTP	                  VP                  VP
                  TVP                  V'       d   VP                  P                  MR RV P                  '       d   TMR /Ve   VP                  4       M\        P                  R7      p VP                  V4      G R j  xL
 p	T'       d   TP)                  T	P*                  4       \-        T	P                  4      Tn        \1        \3        T	4      R7      #  LV  \        P                   d   p
\        P                   T
hR p
?
i\        P"                   d%   p
T P%                  T
4      '       d   \&        T
hh R p
?
ii ; i5i)Nr^   )urlmethodr?   contentrc   ra   r   )http_response)_get_clientr   _combine_headersr?   build_requestr   payloadrc   rf   _persist_cookies_per_sessiontotal_secondsr|   USE_CLIENT_DEFAULTsendTimeoutExceptionasyncioTimeoutErrorTransportError_is_proxy_errorr   register_status_coder:   r2   
loaded_urlr   r   )r.   rZ   rg   r   r   r   clientr?   http_requestr    excs   &&$$$$     r&   crawlHttpxHttpClient.crawl   sH     !!J*..DI''8++>>OO+2GOO'')d6W6W6W7]ab/6/BG))+H`H` , 
	#[[66H ++H,@,@A .!(2
 	
 7%% 	0&&C/## 	##C(( c)	sa   A5F"8)F""-F"D= $D;%D= )F"1A
F";D= =FE##F:F;FFF"r   GETr?   r   c               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r   r   r2   r   r   r?   #HttpHeaders | dict[str, str] | Noner   HttpPayload | Nonerg   r   r   r   r   r   r"   r   r$   )r%   s   "r&   r'   rv      sZ      (  ( ( 	 (
 5 ( $ (   ( % ( " ( 
 (r)   c               	  "   T P                  V'       d   VP                  MR 4      pT P                  TTTTTTVe%   \        P                  ! VP                  4       4      MR R7      p	 VP                  V	4      G R j  xL
 p
\        T
4      #  L  \        P                   d   p\        P                  ThR p?i\        P                   d%   pT P                  T4      '       d   \        Thh R p?ii ; i5i)Nr   r   r   r?   r   rg   r   )r   r   _build_requestr|   Timeoutr   r   r   r   r   r   r   r   r   r.   r   r   r?   r   rg   r   r   r   r   r    r   s   &&$$$$$$    r&   send_requestHttpxHttpClient.send_request   s      !!J*..DI**>E>QEMM'"7"7"9:W[ + 
	#[[66H h'' 7%% 	0&&C/## 	##C(( c)	sN   A&C4)B =B>B C4B C1$B55C1C1C,,C11C4c               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r   r   r2   r   r   r?   r   r   r   rg   r   r   r   r   r   r"   zAsyncGenerator[HttpResponse]r$   )r%   s   "r&   r'   rv      sZ     $ $$ 	$
 5$ $$  $ %$ "$ 
&$r)   c              	
  "   T P                  V'       d   VP                  MR 4      pT P                  TTTTTTV'       d'   \        P                  ! R VP                  4       R7      MR R7      p	 VP                  V	RR7      G R j  xL
 p
 \        T
4      5x  T
P                  4       G R j  xL
  R #  L-  \        P                   d   p\        P                  ThR p?ii ; i L6  T
P                  4       G R j  xL 
  i ; i5i)N)connectr   T)stream)r   r   r   r|   r   r   r   r   r   r   r   acloser   s   &&$$$$$$    r&   r   HttpxHttpClient.stream   s      !!J*..DI**LSEMM$0E0E0GHY] + 
	0#[[d[CCH	$ **//### D%% 	0&&C/	0 $(//###sk   >D-D/B5 B3B5 C% D,C#-D3B5 5C 
CC  D%D 9C<:D  Dc               8    V ^8  d   QhRRRRRRRRR	R
RRRRRR/# )r   r   httpx.AsyncClientr   r2   r   r   r?   r   r   r   rg   r   r   zhttpx.Timeout | Noner"   r[   r$   )r%   s   "r&   r'   rv      sZ     
 
!
 
 	

 5
 $
  
 &
 

r)   c           	     2   \        V\        4      '       g   Vf   \        T;'       g    / 4      pV P                  V4      pTP	                  TTV'       d   \        V4      MRTRV P
                  '       d   TMR/T;'       g    \        P                  R7      # )z7Build an `httpx.Request` using the provided parameters.Nr^   )r   r   r?   r   ra   r   )
isinstancer>   r
   r   r   r   r|   r   )r.   r   r   r   r?   r   rg   r   s   &&&&&&&&r&   r   HttpxHttpClient._build_request   s     gt$$!'--R0G''0##%,DM$)d6W6W6W7]ab77u77 $ 
 	
r)   c                    V ^8  d   QhRRRR/# )r   	proxy_urlz
str | Noner"   r   r$   )r%   s   "r&   r'   rv     s     (4 (4Z (44E (4r)   c                @   V P                   '       ge   V P                  P                  R\        P                  ! R^R7      4      p\        V P                  V P                  V P                  VR7      V n         WP                  9  d   RVRV P                  RV P                  RR	/pVP                  V P                  4       VP                  R
V P                   RV P                  /4       \        P                  ! R/ VB pW@P                  V&   V P                  V,          # )zRetrieve or create an HTTP client for the given proxy URL.

If a client for the specified proxy URL does not exist, create and store a new one.
limitsi  )max_connectionsmax_keepalive_connections)ro   rp   rq   r   proxyro   rp   follow_redirectsT	transportrq   r$   )r   rz   rb   r|   LimitsrX   rx   ry   r~   r   updater   )r.   r   r   kwargsr   s   &&   r&   r   HttpxHttpClient._get_client  s    
 ..22%,,tWZ[F .kkkk((	DO 555 "D	&F MM$334MMd// &&00F39%%i0((33r)   c                    V ^8  d   QhRRRR/# )r   explicit_headerszHttpHeaders | Noner"   r$   )r%   s   "r&   r'   rv   @  s      1C HZ r)   c                8   V P                   '       d   V P                   P                  4       M	\        4       pV P                   '       d   V P                   P                  4       M	\        4       pT;'       g    \        4       pW#,          V,          pT;'       g    R# )zMerge default headers with explicit headers for an HTTP request.

Generate a final set of request headers by combining default headers, a random User-Agent header,
and any explicitly provided headers.
N)r{   get_common_headersr
   get_random_user_agent_header)r.   r   common_headersuser_agent_headerr?   s   &&   r&   r    HttpxHttpClient._combine_headers@  s     IMH^H^H^//BBDdodqEIE[E[E[D""??Aalan 	 ,<<{} 47GG$r)   c                    V ^8  d   QhRRRR/# )r   errorzhttpx.TransportErrorr"   rt   r$   )r%   s   "r&   r'   rv   O  s      3  r)   c                   a  \        S \        P                  4      '       d   R# \        ;QJ d#    V 3R l\         4       F  '       g   K   RM	  RM! V 3R l\         4       4      '       d   R# R# )zDetermine whether the given error is related to a proxy issue.

Check if the error is an instance of `httpx.ProxyError` or if its message contains known proxy-related
error keywords.
Tc              3  >   <"   T F  q\        S4      9   x  K  	  R # 5ir+   )r2   ).0needler   s   & r&   	<genexpr>2HttpxHttpClient._is_proxy_error.<locals>.<genexpr>X  s     F2EU#2Es   F)r   r|   r   anyr   )r   s   fr&   r   HttpxHttpClient._is_proxy_errorN  sD     eU--..3F2EF333F2EFFFr)   c                   V ^8  d   QhRR/# )r   r"   r#   r$   )r%   s   "r&   r'   rv   ]  s     # #t #r)   c                	6  "   V P                   P                  4        F  pVP                  4       G R j  xL
  K  	  V P                   P                  4        V P                  '       d,   V P                  P                  4       G R j  xL
  R V n        R # R #  La L5ir+   )r   valuesr   clearr   )r.   r   s   & r&   cleanupHttpxHttpClient.cleanup]  st     //668F--/!! 9!!'')???//((***"DO  " +s'   1BB3B(BBBB)rz   r   r{   rx   ry   r~   r   )NN)rP   rQ   rR   rS   rT   r   _DEFAULT_HEADER_GENERATORr/   r	   r   r   r   r   r   r   r   staticmethodr   r   rV   rj   rk   s   @r&   rm   rm   U   sm   $ !0 1!J -1!J 	!J
 !J +/!J 4M!J !JF &
 #'	&

 (,&
 )-&
 %)&
 &
P  ( #	 (
 8< ( '+ ( #' ( (, ( %) (  (D $ #	$
 8<$ '+$ #'$ (,$ %)$  $B
2(4T  # #r)   rm   )3
__future__r   r   
contextlibr   loggingr   typingr   r   r   r|   typing_extensionsr	   crawlee._typesr
   crawlee._utils.blockedr   crawlee._utils.docsr   crawlee.errorsr   crawlee.fingerprint_suiter   crawlee.http_clientsr   r   r   collections.abcr   r   datetimer   sslr   crawleer   r   r   crawlee.proxy_configurationr   crawlee.sessionsr   crawlee.statisticsr   rP   loggerr   AsyncHTTPTransportrX   rm   r$   r)   r&   <module>r      s    "  *  + +  & & 6 * % 5 M M="65(-	8	 >e.. , NM#j M# M#r)   