+
    }i$                        ^ RI Ht ^ RIt^ RIHt ^RIHtHtHt ]'       d   ^ RI	H
t
 ^ RIHt ]P                  ! ]4      t0 RmtR0t0 Rmt0 RmtRR	0tR
R0tRR0tR0tR R lt ! R R4      tR# )    )annotationsN)TYPE_CHECKING)RequestRate	VisitTime_RuleSet)Iterator)Selfallowzcrawl-delayzcrawl delayzrequest-ratezrequest ratez
visit-timez
visit timehostc                    V ^8  d   QhRRRR/# )   fieldstrreturnbool )formats   "b/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/protego/_protego.py__annotate__r   #   s      S T     c           
         \        V \        9   V \        9   V \        9   V \        9   V \
        9   V \        9   V \        9   .4      # N)any_DISALLOW_DIRECTIVE_ALLOW_DIRECTIVE_USER_AGENT_DIRECTIVE_SITEMAP_DIRECTIVE_CRAWL_DELAY_DIRECTIVE_REQUEST_RATE_DIRECTIVE_HOST_DIRECTIVE)r   s   &r   _is_valid_directive_fieldr!   #   sR    ((%%**''++,,_$	

 
r   c                      ] tR t^1tR R lt]R R l4       tR R ltR R ltR	 R
 lt	R R lt
R R ltR R lt]R R l4       t]R R l4       t]R R l4       tRtR# )Protegoc                   V ^8  d   QhRR/# )r   r   Noner   )r   s   "r   r   Protego.__annotate__2   s     ' '$ 'r   c                	h    / V n         R V n        . V n        / V n        ^ V n        ^ V n        ^ V n        R # r   )_user_agents_host_sitemap_list_matched_rule_set_total_line_seen_invalid_directive_seen_total_directive_seenselfs   &r   __init__Protego.__init__2   s@    13 "&
 )+ >@ !'($%&"r   c                    V ^8  d   QhRRRR/# )r   contentr   r   r	   r   )r   s   "r   r   r&   D   s      C D r   c                	    V ! 4       p\        V\        4      '       g"   \        R \        V4      P                   24      hVP                  V4       V# )zProtego.parse expects str, got )
isinstancer   
ValueErrortype__name___parse_robotstxt)clsr4   os   && r   parseProtego.parseC   sG    E'3''>tG}?U?U>VWXX	7#r   c                    V ^8  d   QhRRRR/# )r   r4   r   r   r%   r   )r   s   "r   r   r&   K   s     {& {& {& {&r   c                	   VP                  4       p. pR pV EF  pV ;P                  ^,          un        VP                  R4      pVR8w  d   V^ V P                  4       pVP                  4       pV'       g   Kb  VP                  R4      R8w  d   VP	                  R^4      w  rxMVP	                  R4      p	\        V	4      ^8  d   K  V	^ ,          p
\        ^\        V	4      4       F@  p\        V
4      '       d   T
RP                  WR  4      r MV
RW,          ,           ,          p
KB  	  EK  VP                  4       P                  4       pVP                  4       pV'       g   TpEKP  V'       g=   V\        9  d2   V\        9  d'   \        P                  RV P                   R24       EK  V ;P                  ^,          un        V\        9   d   V'       d   V\        9  d   . pVP                  4       P                  4       pR pVR8w  d   RV9   d   VP                  RR4      pW.pV F  pV'       g   K  V P                   P#                  VR 4      pV'       d   VV9  d   VP%                  V4       V'       d   KS  \'        V 4      pVVn        VV P                   V&   VP%                  V4       K  	  MV\*        9   d   V F  pVP-                  V4       K  	  MV\.        9   d   V F  pVP1                  V4       K  	  MV\        9   d   V P2                  P%                  V4       MV\4        9   d   V F
  pVVn        K  	  McV\8        9   d   V F
  pVVn        K  	  MGV\<        9   d   Wn        M5V\@        9   d   V F
  pVVn!        K  	  MV ;PD                  ^,          un"        TpEK  	  V P                   PG                  4        F  pVPI                  4        K  	  R # )	N#: zRule at line z) without any user agent to enforce it on.* )%
splitlinesr,   findstripsplitlenranger!   joinlowerr   r   loggerdebugr.   replacer(   getappendr   
user_agentr   r
   r   disallowr*   r   crawl_delayr   request_rater    r)   _VISIT_TIME_DIRECTIVE
visit_timer-   valuesfinalize_rules)r0   r4   linescurrent_rule_setsprevious_rule_fieldlinehash_posr   valuepartspossible_filedirT   user_agent_without_asteriskuser_agentsuarule_sets   &&               r   r:   Protego._parse_robotstxtK   sh   ""$ -/ #D!!Q&! yy~H2~Ah'--/ ::<D yy~##zz#q1u 

3u:>!&qq#e*-A0@@'5sxxb	7Ju"cEHn4N	 . KKM'')EKKME &+# &!66!33#D$9$9#::cd &&!+&--'+3HH(*% #[[]002
.2+$
):2<2D2DS"2M/)G%B #0044R>HH4E$E)00:#8#+D>.0+08))"-)00: & ** 1HNN5) !2 -- 1H%%e, !2 ,,""))%000 1H+0H( !2 11 1H,1H) !2 /)"
// 1H*/H' !2 ,,1,"'] ` ))002H##% 3r   c                    V ^8  d   QhRRRR/# )r   rT   r   r   z_RuleSet | Noner   )r   s   "r   r   r&      s            r   c                .  a V P                   '       g   R# SV P                  9   d   V P                  S,          # V3R lV P                   P                  4        4       p\        VR R7      w  r4V'       g   RV P                  S&   R# W@P                  S&   V# )z0Return the rule set with highest matching score.Nc              3  H   <"   T F  qP                  S4      V3x  K  	  R # 5ir   )
applies_to).0rsrT   s   & r   	<genexpr>1Protego._get_matching_rule_set.<locals>.<genexpr>   s!       
6P]]:&+6Ps   "c                    V ^ ,          # )r   r   )ps   &r   <lambda>0Protego._get_matching_rule_set.<locals>.<lambda>   s    PQRSPTr   )key)r(   r+   rZ   max)r0   rT   score_rule_set_pairsmatch_scorematched_rule_sets   &f   r   _get_matching_rule_setProtego._get_matching_rule_set   s       ///))*55 
6:6G6G6N6N6P 
 ),,@n(U%15D"":.-=z*r   c               $    V ^8  d   QhRRRRRR/# )r   urlr   rT   r   r   r   )r   s   "r   r   r&      s!     / /S /c /d /r   c                Z    V P                  V4      pV'       g   R# VP                  V4      # )zHReturn True if the user agent can fetch the URL, otherwise return False.T)r{   	can_fetch)r0   r~   rT   rz   s   &&& r   r   Protego.can_fetch   s*    66zB))#..r   c                    V ^8  d   QhRRRR/# )r   rT   r   r   zfloat | Noner   )r   s   "r   r   r&      s     , ,c ,l ,r   c                P    V P                  V4      pV'       g   R# VP                  # )zfReturn the crawl delay specified for the user agent as a float.
If nothing is specified, return None.
N)r{   rV   r0   rT   rz   s   && r   rV   Protego.crawl_delay   s'      66zB+++r   c                    V ^8  d   QhRRRR/# )r   rT   r   r   zRequestRate | Noner   )r   s   "r   r   r&      s     - -s -/A -r   c                P    V P                  V4      pV'       g   R# VP                  # )zReturn the request rate specified for the user agent as a named tuple
RequestRate(requests, seconds, start_time, end_time). If nothing is
specified, return None.
N)r{   rW   r   s   && r   rW   Protego.request_rate   s'    
  66zB,,,r   c                    V ^8  d   QhRRRR/# )r   rT   r   r   zVisitTime | Noner   )r   s   "r   r   r&      s     + +S +-= +r   c                P    V P                  V4      pV'       g   R# VP                  # )zReturn the visit time specified for the user agent as a named tuple
VisitTime(start_time, end_time). If nothing is specified, return None.
N)r{   rY   r   s   && r   rY   Protego.visit_time   s'      66zB***r   c                   V ^8  d   QhRR/# )r   r   zIterator[str]r   )r   s   "r   r   r&      s     ( (- (r   c                ,    \        V P                  4      # )z7Get an iterator containing links to sitemaps specified.)iterr*   r/   s   &r   sitemapsProtego.sitemaps   s     D&&''r   c                   V ^8  d   QhRR/# )r   r   z
str | Noner   )r   s   "r   r   r&     s      
 r   c                    V P                   # )zGet the preferred host.)r)   r/   s   &r   preferred_hostProtego.preferred_host  s     zzr   c                   V ^8  d   QhRR/# )r   r   intr   )r   s   "r   r   r&     s     I Is Ir   c                	<    V P                   V P                  ,
          # r   )r.   r-   r/   s   &r   _valid_directive_seenProtego._valid_directive_seen  s    ))D,H,HHHr   )r)   r-   r+   r*   r.   r,   r(   N)r9   
__module____qualname____firstlineno__r1   classmethodr=   r:   r{   r   rV   rW   rY   propertyr   r   r   __static_attributes__r   r   r   r#   r#   1   sy    '"  {&z $/,-+ ( (   I Ir   r#   >   disalowdiasllowdisallawrU   dissalow	dissallow>   	useragent
user agent
user-agent>   sitemapr   site-map)
__future__r   loggingtypingr   _rulesetr   r   r   collections.abcr   typing_extensionsr	   	getLoggerr9   rO   r   r   r   r   r   r   rX   r    r!   r#   r   r   r   <module>r      s    "    6 6( ' 
		8	$  9 A 8 '7 )>: %|4 (XI XIr   