+
    }i                        ^ RI Ht ^ RIt^ RIHtHt ^RIHt ^RIH	t	H
t
HtHt ]'       d   ^ RIHt ^RIHt ]P"                  ! ]4      t ! R R	]4      t ! R
 R]4      t ! R R]4      t ! R R4      tR# )    )annotationsN)TYPE_CHECKING
NamedTuple)_URLPattern)
_hexescape_parse_time_period_quote_path_quote_pattern)time)Protegoc                  @    ] tR t^t$ R]R&   R]R&   R]R&   R]R&   RtR# )	RequestRateintrequestssecondsztime | None
start_timeend_time N__name__
__module____qualname____firstlineno____annotations____static_attributes__r       b/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/protego/_ruleset.pyr   r      s    MLr   r   c                  ,    ] tR t^t$ R]R&   R]R&   RtR# )	VisitTimer   r   r   r   Nr   r   r   r   r   r      s    Nr   r   c                  ,    ] tR t^t$ R]R&   R]R&   RtR# )_Rulestrfieldr   valuer   Nr   r   r   r   r!   r!      s    Jr   r!   c                  "   ] tR t^#tRtR R ltR R ltR R ltR R	 ltR
 R lt	R R lt
]R R l4       t]P                  R R l4       t]R R l4       t]P                  R R l4       t]R R l4       t]P                  R R l4       tRtR# )_RuleSetz3Internal class which stores rules for a user agent.c                   V ^8  d   QhRR/# )   parser_instancer   r   )formats   "r   __annotate___RuleSet.__annotate__&   s     9 9 9r   c                	X    R V n         . V n        R V n        R V n        R V n        Wn        R # )N)
user_agent_rules_crawl_delay	_req_rate_visit_time_parser_instance)selfr)   s   &&r   __init___RuleSet.__init__&   s-    &*#%*.-1-1)8r   c                    V ^8  d   QhRRRR/# )r(   	robotnamer"   returnr   r   )r*   s   "r   r+   r,   .   s      C C r   c                    V P                   f   Q hVP                  4       P                  4       pV P                   R8X  d   ^# V P                   V9   d   \        V P                   4      # ^ # )zReturn matching score.*)r.   striplowerlen)r4   r8   s   &&r   
applies_to_RuleSet.applies_to.   sT    ***OO%++-	??c!??i't''r   c                    V ^8  d   QhRRRR/# r(   patternr"   r9   Noner   )r*   s   "r   r+   r,   8   s     , ,S ,T ,r   c                	X   R V9   d+   V P                  VP                  R \        R 4      4      4       \        V4      pV'       g   R# V P                  P                  \        R\        V4      R7      4       VP                  R4      '       d   V P                  VRR R ,           4       R# R# )$Nallowr#   r$   z/index.htmli)	rG   replacer   r
   r/   appendr!   r   endswithr4   rC   s   &&r   rG   _RuleSet.allow8   s    '>JJwsJsO<= )5wk'6JKL M**JJwt}s*+ +r   c                    V ^8  d   QhRRRR/# rB   r   )r*   s   "r   r+   r,   E   s     P P P Pr   c                	    R V9   d+   V P                  VP                  R \        R 4      4      4       \        V4      pV'       g   R# V P                  P                  \        R\        V4      R7      4       R# )rF   NdisallowrH   )rP   rI   r   r
   r/   rJ   r!   r   rL   s   &&r   rP   _RuleSet.disallowE   sR    '>MM'//#z#?@ )5zW9MNOr   c                   V ^8  d   QhRR/# )r(   r9   rD   r   )r*   s   "r   r+   r,   N   s     
 
 
r   c                	B    V P                   P                  R  RR7       R# )c                L    V P                   P                  V P                  R 8H  3# )rG   )r$   priorityr#   )rs   &r   <lambda>)_RuleSet.finalize_rules.<locals>.<lambda>P   s    177++QWW-?@r   T)keyreverseN)r/   sortr4   s   &r   finalize_rules_RuleSet.finalize_rulesN   s    @$ 	 	
r   c                    V ^8  d   QhRRRR/# )r(   urlr"   r9   boolr   )r*   s   "r   r+   r,   S   s     	 	S 	T 	r   c                    \        V4      pRpV P                   F:  pVP                  P                  V4      '       g   K&  VP                  R8X  d   Rp V# 	  V# )z!Return if the url can be fetched.TrP   F)r	   r/   r$   matchr#   )r4   r`   allowedrules   &&  r   	can_fetch_RuleSet.can_fetchS   sT    #KKDzz$$::+#G  
 r   c                   V ^8  d   QhRR/# )r(   r9   zfloat | Noner   )r*   s   "r   r+   r,   _   s     ! !\ !r   c                    V P                   # )z'Get & set crawl delay for the rule set.)r0   r\   s   &r   crawl_delay_RuleSet.crawl_delay^   s        r   c                    V ^8  d   QhRRRR/# )r(   delayr"   r9   rD   r   )r*   s   "r   r+   r,   d   s        r   c                	     \        V4      V n        R#   \         d5    \        P	                  R T P
                  P                   RT R24        R# i ; i)Malformed rule at line z : cannot set crawl delay to ''. Ignoring this rule.N)floatr0   
ValueErrorloggerdebugr3   _total_line_seen)r4   rm   s   &&r   rj   rk   c   sV    	 %eD 	LL)$*?*?*P*P)Q R..3W4JL	s    ;AAc                   V ^8  d   QhRR/# )r(   r9   zRequestRate | Noner   )r*   s   "r   r+   r,   o   s      0 r   c                    V P                   # )z(Get & set request rate for the rule set.)r1   r\   s   &r   request_rate_RuleSet.request_raten   s     ~~r   c                    V ^8  d   QhRRRR/# r(   r$   r"   r9   rD   r   )r*   s   "r   r+   r,   t   s     N N# N$ Nr   c                	&    VP                  4       p\        V4      ^8X  d   Vw  r4M
V^ ,          RrCVP                  R4      w  rVVR,          P                  4       p\        V4      \        VRR 4      rVR8X  d   V	^<,          p	M!VR8X  d   V	R,          p	MVR8X  d
   V	R,          p	Rp
RpV'       d   \	        V4      w  r\        YY4      T n        R#   \
         d5    \        P                  R	T P                  P                   R
T R24        R# i ; i)r(    /Nmhi  diQ ro   z" : cannot set request rate using 'rp   )splitr>   r=   r   r   	Exceptionrs   rt   r3   ru   r   r1   )r4   r$   partsratetime_periodrequests_strseconds_str	time_unitr   r   r   r   s   &&          r   rx   ry   s   s   	KKME5zQ$)!k$)!Hbk(,

3%L#B--/I #L 13{3B7G3HgC2c!4c!5 JH'9+'F$
 %X
M  	LL)$*?*?*P*P)Q R2278NP 	s   B.C 1C ;DDc                   V ^8  d   QhRR/# )r(   r9   zVisitTime | Noner   )r*   s   "r   r+   r,      s        ,  r   c                    V P                   # )z&Get & set visit time for the rule set.)r2   r\   s   &r   
visit_time_RuleSet.visit_time   s     r   c                    V ^8  d   QhRRRR/# r{   r   )r*   s   "r   r+   r,      s     	; 	; 	; 	;r   c                	     \        VR R7      w  r#\        Y#4      T n        R#   \         d5    \        P                  RT P                  P
                   RT R24        R# i ; i) )	separatorro   z  : cannot set visit time using 'rp   N)r   r   rs   rt   r3   ru   r   r2   )r4   r$   r   r   s   &&  r   r   r      so    	#5es#K J %Z:  	LL)$*?*?*P*P)Q R005w6LN 	s   # ;A"!A")r0   r3   r1   r/   r2   r.   N)r   r   r   r   __doc__r5   r?   rG   rP   r]   rf   propertyrj   setterrx   r   r   r   r   r   r&   r&   #   s    =9,P

	 ! !     N NB     	; 	;r   r&   )
__future__r   loggingtypingr   r   _urlpatternr   _utilsr   r   r	   r
   datetimer   _protegor   	getLoggerr   rs   r   r   r!   r&   r   r   r   <module>r      sb    "  , $ O O! 
		8	$* 
 
J 
A; A;r   