+
    }i/                       ^ RI Ht ^ RIt^ RIt^ RIHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^ R	IHt ]'       d   ^ R
IHtHt ^ RIHt ]	! ]4      t ! R R]4      t ! R R4      t]! R4       ! R R4      4       tR# )    )annotationsN)suppress)	timedelta)	getLogger)TYPE_CHECKING)ConcurrencySettings)
docs_group)RecurringTask)	AwaitableCallable)SystemStatusc                      ] tR t^tRtRtR# )
AbortErrorzARaised when an AutoscaledPool run is aborted. Not for direct use. N)__name__
__module____qualname____firstlineno____doc____static_attributes__r       v/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/crawlee/_autoscaling/autoscaled_pool.pyr   r      s    Kr   r   c                  "    ] tR t^tR R ltRtR# )_AutoscaledPoolRunc                   V ^8  d   QhRR/#    returnNoner   )formats   "r   __annotate___AutoscaledPoolRun.__annotate__   s     7 7$ 7r   c                	    \         \        P                  ,          ! 4       V n         \        P                  ! 4       V n        \        P                  ! 4       V n        \        P                  ! 4       V n        R # N)	listasyncioTaskworker_tasksEventworker_tasks_updatedcleanup_doneFutureresultselfs   &r   __init___AutoscaledPoolRun.__init__   sC     .0:$+MMO!#MMO&-nn&6r   )r+   r-   r(   r*   N)r   r   r   r   r0   r   r   r   r   r   r      s    7 7r   r   Autoscalingc                     ] tR t^&t$ Rt]! ^
R7      t ]! ^R7      t Rt Rt	 Rt
 RtR]R&    R	R/R
 R lltR R ltR R ltR R ltR R lt]R R l4       t]R R l4       tR R ltR R ltR R ltR R ltR  R! ltR"tR# )#AutoscaledPoola  Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.

The pool only starts new tasks if there is enough free CPU and memory available. If an exception is thrown in
any of the tasks, it is propagated and the pool is stopped.
)seconds)minutesg?g?Nztimedelta | None_TASK_TIMEOUTconcurrency_settingsc               0    V ^8  d   QhRRRRRRRRR	RR
R/# )r   system_statusr   r8   zConcurrencySettings | Nonerun_task_functionzCallable[[], Awaitable]is_task_ready_functionzCallable[[], Awaitable[bool]]is_finished_functionr   r   r   )r    s   "r   r!   AutoscaledPool.__annotate__@   sH     '< '< $'< 9	'<
 3'< !>'< <'< 
'<r   c                  T;'       g    \        4       pWn        W0n        W@n        WPn        VP
                  V n        VP                  V n        VP                  V n
        VP                  V n        \        V P                  V P                  4      V n        \        V P"                  V P$                  4      V n        RV n        RV n        R# )a  Initialize a new instance.

Args:
    system_status: Provides data about system utilization (load).
    concurrency_settings: Settings of concurrency levels.
    run_task_function: A function that performs an asynchronous resource-intensive task.
    is_task_ready_function: A function that indicates whether `run_task_function` should be called. This
        function is called every time there is free capacity for a new task and it should indicate whether
        it should start a new task or not by resolving to either `True` or `False`. Besides its obvious use,
        it is also useful for task throttling to save resources.
    is_finished_function: A function that is called only when there are no tasks to be processed. If it
        resolves to `True` then the pool's run finishes. Being called only when there are no tasks being
        processed means that as long as `is_task_ready_function` keeps resolving to `True`,
        `is_finished_function` will never be called. To abort a run, use the `abort` method.
FN)r   _system_status_run_task_function_is_task_ready_function_is_finished_functiondesired_concurrency_desired_concurrencymax_concurrency_max_concurrencymin_concurrency_min_concurrencymax_tasks_per_minute_max_tasks_per_minuter
   _log_system_status_LOGGING_INTERVAL_log_system_status_task
_autoscale_AUTOSCALE_INTERVAL_autoscale_task
_is_paused_current_run)r/   r:   r8   r;   r<   r=   s   &$$$$$r   r0   AutoscaledPool.__init__@   s    0  4LL7J7L+"3'=$%9"$8$L$L! 4 D D 4 D D%9%N%N"'4T5L5LdNdNd'e$,T__d>V>VW7;r   c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>   i   s     02 024 02r   c           
     	  "   V P                   e   \        R4      h\        4       pWn         \        P	                  R4       V P
                  P                  4        V P                  P                  4        \        P                  ! V P                  V4      RR7      p VP                  G Rj  xL
  \!        \        P"                  4      ;_uu_ 4        T P
                  P%                  4       G Rj  xL
  RRR4       \!        \        P"                  4      ;_uu_ 4        T P                  P%                  4       G Rj  xL
  RRR4       TP                  4       '       g   TP                  4        MMTP'                  4       '       g8   TP)                  4       e&   \        P+                  RTP)                  4       R7       \        P-                  R4       TP                   FE  pTP                  4       '       d   K  \!        \.        4      ;_uu_ 4        TG Rj  xL
  RRR4       KG  	  TP0                  P3                  4        RT n         \        P	                  R	4       R#  EL  \         dP    TP                  4        TP                   F+  pTP                  4       '       d   K  TP                  4        K-  	   EL
i ; i EL  + '       g   i     EL; i EL  + '       g   i     EL; i L  + '       g   i     EK#  ; i  \!        \        P"                  4      ;_uu_ 4        T P
                  P%                  4       G Rj  xL 
  RRR4       M  + '       g   i     M; i\!        \        P"                  4      ;_uu_ 4        T P                  P%                  4       G Rj  xL 
  RRR4       M  + '       g   i     M; iTP                  4       '       g   TP                  4        MMTP'                  4       '       g8   TP)                  4       e&   \        P+                  RTP)                  4       R7       \        P-                  R4       TP                   FZ  pTP                  4       '       d   K  \!        \.        4      ;_uu_ 4        TG Rj  xL 
  RRR4       KH    + '       g   i     KY  ; i	  TP0                  P3                  4        RT n         \        P	                  R	4       i ; i5i)
zStart the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True.

If there is an exception in one of the tasks, it will be re-raised.
NzThe pool is already runningzStarting the poolz(autoscaled pool worker task orchestratornamez%Exception in worker task orchestrator)exc_infoz%Waiting for remaining tasks to finishzPool cleanup finished)rS   RuntimeErrorr   loggerdebugrQ   startrN   r&   create_task_worker_task_orchestratorr-   r   cancelr(   doner   CancelledErrorstop	cancelled	exceptionerrorinfoBaseExceptionr+   set)r/   runorchestratortasks   &   r   rj   AutoscaledPool.runi   s]    
 (<== "()""$$$**,****3/6`
	2** '0011**//111 2'00112277999 2  $$&&##%!++--,2H2H2J2VD|OeOeOghKK?@((yy{{!-00"

 10 )
   " $DLL017  	"!((yy{{KKM )	" 2 211 : 211 # 100 '0011**//111 2111'00112277999 2111  $$&&##%!++--,2H2H2J2VD|OeOeOghKK?@((yy{{!-00"

 1000 )
   " $DLL01s`  BS4I" ,I-I" 1!S4K0J?1K5)S4K<K=KS4&S4A.S49S4K/K-
K/AS4I" "=J<$J<8L ;J<<L ?KK		S4KK*	$	S4-K//L:
S4"S1&M	MM	
	S1M#)S1N4	%N(&N4	+	S14O?S1&S1A.S17S1R$RR$
S1$R5/AS11S4c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>      s     4 4T 4r   c                   "   V P                   '       g   \        R4      hV P                   P                  P                  \	        4       4       V P                   P
                  P                  4       G Rj  xL
  R#  L5i)z<Interrupt the autoscaled pool and all the tasks in progress.zThe pool is not runningN)rS   rZ   r-   set_exceptionr   r+   waitr.   s   &r   abortAutoscaledPool.abort   sW        899  ..z|<,,11333s   A2A=4A;5A=c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>      s      t r   c                    RV n         R# )z>Pause the autoscaled pool so that it does not start new tasks.TNrR   r.   s   &r   pauseAutoscaledPool.pause   s	    r   c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>      s          r   c                    RV n         R# )zHResume a paused autoscaled pool so that it continues starting new tasks.FNrv   r.   s   &r   resumeAutoscaledPool.resume   s	    r   c                   V ^8  d   QhRR/# r   r   intr   )r    s   "r   r!   r>      s     ) )S )r   c                    V P                   # )zWThe current desired concurrency, possibly updated by the pool according to system load.)rE   r.   s   &r   rD   "AutoscaledPool.desired_concurrency   s     (((r   c                   V ^8  d   QhRR/# r~   r   )r    s   "r   r!   r>      s     3 3S 3r   c                `    V P                   f   ^ # \        V P                   P                  4      # )z+The number of concurrent tasks in progress.)rS   lenr(   r.   s   &r   current_concurrency"AutoscaledPool.current_concurrency   s*     $4$$1122r   c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>      s     e eD er   c                $   V P                   P                  4       p\        P                  ! V P                  V P
                  ,          4      pVP                  ;'       d1    V P                  V P                  8  ;'       d    V P                  V8  pVP                  '       * ;'       d    V P                  V P                  8  pV'       d`   \        P                  ! V P                  V P                  ,          4      p\        V P                  V P                  V,           4      V n        R# V'       d`   \        P                  ! V P                  V P                  ,          4      p\        V P                  V P                  V,
          4      V n        R# R# )z]Inspect system load status and adjust desired concurrency if necessary. Do not call directly.N)r@   get_historical_system_infomathfloor_DESIRED_CONCURRENCY_RATIOrD   is_system_idlerE   rG   r   rI   ceil_SCALE_UP_STEP_RATIOmin_SCALE_DOWN_STEP_RATIOmax)r/   statusmin_current_concurrencyshould_scale_upshould_scale_downsteps   &     r   rO   AutoscaledPool._autoscale   s5   $$??A"&**T-L-LtOgOg-g"h!! D D))D,A,AAD D((,CC 	 !' 5 55kk$:S:SVZVkVk:k99T669R9RRSD(+D,A,A4C\C\_cCc(dD%99T884;T;TTUD(+D,A,A4C\C\_cCc(dD% r   c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>      s     
 
D 
r   c                	    V P                   P                  4       p\        P                  R V P                   RV P
                   RV: 24       R# )zcurrent_concurrency = z; desired_concurrency = z; N)r@   r   r[   rg   r   rD   )r/   r:   s   & r   rL   !AutoscaledPool._log_system_status   sN    ++FFH$T%=%=$> ?%%)%=%=$>b!	
r   c                    V ^8  d   QhRRRR/# )r   rj   r   r   r   r   )r    s   "r   r!   r>      s     /0 /03E /0$ /0r   c                
  a a"   Rp S P                  4       G Rj  xL
 ;p'       Eg6   SP                  P                  4       '       Eg   SP                  P	                  4        S P
                  P                  4       pVP                  '       g   \        P                  R4       EMRS P                  '       d   \        P                  R4       EM)S P                  S P                  8  d   \        P                  R4       MS P                  4       G Rj  xL
 '       g   \        P                  R4       M\        P                  R4       \        P                  ! S P!                  4       RR	7      pVP#                  VV 3R
 l4       SP$                  P'                  V4       \(        P*                  ! S P,                  4      '       d0   \        P.                  ! ^<S P,                  ,          4      G Rj  xL
  EK  \1        \        P2                  4      ;_uu_ 4        \        P4                  ! SP                  P7                  4       RR7      G Rj  xL
  RRR4       EKV  V'       d   \        P                  R4       MQSP                  P                  4       '       d2   SP                  P9                  4       e   \        P                  R4       SP$                  '       dd   \        P                  R4       \        P6                  ! SP$                  \        P:                  R7      G Rj  xL
  \        P                  R4       M\        P                  R4       SP                  P                  4       '       g&   SP                  P=                  \?        4       4       R# R#  EL EL EL ELY  + '       g   i     EK  ; i L  T'       d   \        P                  R4       MQSP                  P                  4       '       d2   SP                  P9                  4       e   \        P                  R4       SP$                  '       de   \        P                  R4       \        P6                  ! SP$                  \        P:                  R7      G Rj  xL 
  \        P                  R4       M\        P                  R4       SP                  P                  4       '       g%   SP                  P=                  \?        4       4       i i ; i5i)zzLaunch worker tasks whenever there is free capacity and a task is ready.

Exits when `is_finished_function` returns True.
FNz/Not scheduling new tasks - system is overloadedz8Not scheduling new tasks - the autoscaled pool is pausedzANot scheduling new tasks - already running at desired concurrencyz*Not scheduling new task - no task is readyzScheduling a new taskzautoscaled pool worker taskrW   c                (   < SP                  V S4      # r$   )_reap_worker_task)rl   rj   r/   s   &r   <lambda>:AutoscaledPool._worker_task_orchestrator.<locals>.<lambda>   s    t?U?UVZ\_?`r   g      ?timeoutz3`is_finished_function` reports that we are finishedz*Unhandled exception in `run_task_function`z+Terminating - waiting for tasks to complete)return_whenzWorker tasks finishedz*Terminating - no running tasks to wait for) rC   r-   ra   r*   clearr@   get_current_system_infor   r[   r\   rR   r   rD   rB   r&   r^   _worker_taskadd_done_callbackr(   appendr   isfiniterK   sleepr   TimeoutErrorwait_forrq   re   ALL_COMPLETED
set_resultobject)r/   rj   finishedcurrent_statusworker_tasks   ff   r   r_   (AutoscaledPool._worker_task_orchestrator   s7    
 (	0)-)C)C)E#EExEszzO`O`((..0!%!4!4!L!L!N%444LL!RS___LL![\--1I1IILL!de#;;===LL!MNLL!89")"5"5d6G6G6IPm"nK112`a$$++K8}}T%?%?@@%mmB1K1K,KLLLg2233!**3+C+C+H+H+JTWXXX 43 RS""szz';';'='IIJJKll3#3#3AVAVWWW45IJ::??$$

%%fh/ %M $F > M
 Y 433 X RS""szz';';'='IIJJKll3#3#3AVAVWWW45IJ::??$$

%%fh/ %s   TO NO  O 
AO (O 9AO N!	O B'O *O 1N$2(O 4N*N'N*O =TATAT'N?(AT7'TO !O $O 'N**N<	5
O ?T>T ATATRAT%TTc               $    V ^8  d   QhRRRRRR/# )r   rl   zasyncio.Taskrj   r   r   r   r   )r    s   "r   r!   r>     s#     0 0l 09K 0PT 0r   c                N   VP                   P                  4        VP                  P                  V4       VP	                  4       '       gZ   VP                  4       ;p'       d@   VP                  P                  4       '       g   VP                  P                  V4       R# R# R# R# )zHandle cleanup and tracking of a completed worker task.

- Interrupt the run if the task encountered an exception.
- Update the list of tasks in progress.
- Notify the orchestrator about the task completion.
N)	r*   ri   r(   removerd   re   r-   ra   rp   )r/   rl   rj   re   s   &&& r   r    AutoscaledPool._reap_worker_task  sw     	  $$&%~~$..2B%BY%BCJJOOL]L]JJ$$Y/ M^%Br   c                   V ^8  d   QhRR/# r   r   )r    s   "r   r!   r>     s     
1 
1D 
1r   c                	  "    \         P                  ! V P                  4       V P                  e   V P                  P	                  4       MR R7      G R j  xL
  \        P                  R4       R #  L  \         P
                   dG    T P                  e   T P                  P	                  4       MRp\        P                  RT R24        Lsi ; i  \        P                  R4       i ; i5i)Nr   z	*not set*zTask timed out after z secondszWorker task finished)	r&   r   rA   r7   total_secondsr   r[   warningr\   )r/   timeout_strs   & r   r   AutoscaledPool._worker_task  s     		1""'')>B>P>P>\**88:bf   LL/0 ## 	J@D@R@R@^$,,::<doKNN2;-xHI	J LL/0sH   C+AA3 A1A3 C+1A3 3ACC CC C((C+)rQ   rS   rE   rC   rR   rB   rN   rG   rK   rI   rA   r@   )r   r   r   r   r   r   rP   rM   r   r   r   r7   __annotations__r0   rj   rr   rw   r{   propertyrD   r   rO   rL   r_   r   r   r   r   r   r   r4   r4   &   s     $B/r!!,G!$eP!U&*M#*E'< <@	'<R02d4  ) ) 3 3e(
/0b0
1 
1r   r4   )
__future__r   r&   r   
contextlibr   datetimer   loggingr   typingr   crawlee._typesr   crawlee._utils.docsr	   crawlee._utils.recurring_taskr
   collections.abcr   r   crawlee._autoscalingr   r   r[   	Exceptionr   r   r4   r   r   r   <module>r      sq    #        . * 731	8	L L7 7 Mv1 v1 v1r   