
    Iio                     "   S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJrJrJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJrJrJrJr  S SKJrJr  S S	KJr  S S
KJ r J!r!J"r"J#r#J$r$J%r%  SSK&J'r'J(r(J)r)J*r*  SSK+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5  SSK6J7r7J8r8J9r9J:r:J;r;J<r<J=r=J>r>J?r?  \(       a  SSKJ@r@JArAJBrBJCrC  \R                  " S5      rE\F" 5       rG\ " S S5      5       rHSSSSSS.S\\I\HS4   S\\I   S\S   S\S   S\\J   S\\K   S\H4S jjrL " S S\M5      rN " S  S!\M5      rO " S" S#\M5      rP " S$ S%\M5      rQ " S& S'5      rRS(S)S\\I\S4   4S* jrTS(S)S+\S\R4S, jrUS(S)S+\S\R4S- jrV S3S.\\(   S/\\I   S0\\I   S\K4S1 jjrWS(S)S\K4S2 jrXg)4    N)	dataclass)datetime	timedeltatimezone)partial)Signals)time)TYPE_CHECKINGAnyCallableDictListOptionalSequenceSetTupleUnioncast)ResponseError
WatchError)CronJob)Deserializer	JobResultSerializationError
Serializerdeserialize_job_rawserialize_result   )ArqRedisRedisSettingscreate_poollog_redis_info)
abort_job_max_ageabort_jobs_ssdefault_queue_nameexpires_extra_mshealth_check_key_suffixin_progress_key_prefixjob_key_prefixkeep_cronjob_progressresult_key_prefixretry_key_prefix)	args_to_stringimport_stringms_to_datetimepolltimestamp_msto_ms
to_seconds
to_unix_mstruncate)SecondsTimedeltaStartupShutdownWorkerCoroutineWorkerSettingsTypez
arq.workerc                   j    \ rS rSr% \\S'   S\S'   \\   \S'   \\   \S'   \\   \S'   \\	   \S'   S	r
g
)Function2   namer8   	coroutine	timeout_skeep_result_skeep_result_forever	max_tries N)__name__
__module____qualname____firstlineno__str__annotations__r   floatboolint__static_attributes__rC       0/venv/lib/python3.13/site-packages/arq/worker.pyr;   r;   2   s5    
I  E?"!$'}rN   r;   )r=   keep_resulttimeoutrA   rB   r>   r8   r=   rP   r6   rQ   rA   rB   returnc                L   [        U [        5      (       a  U $ [        U [        5      (       a  U=(       d    U n[        U 5      nOU n[        R
                  " U5      (       d  [        U S35      e[        U5      n[        U5      n[        U=(       d    UR                  XcX$U5      $ )a,  
Wrapper for a job function which lets you configure more settings.

:param coroutine: coroutine function to call, can be a string to import
:param name: name for function, if None, ``coroutine.__qualname__`` is used
:param keep_result: duration to keep the result for, if 0 the result is not kept
:param keep_result_forever: whether to keep results forever, if None use Worker default, wins over ``keep_result``
:param timeout: maximum time the job should take
:param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying
z is not a coroutine function)	
isinstancer;   rH   r.   asyncioiscoroutinefunctionRuntimeErrorr3   rF   )r>   r=   rP   rQ   rA   rB   
coroutine_s          rO   funcrY   <   s    & )X&&)S!! y(5i(@

&&z22j\)EFGG!G[)KD3J33Z+luvvrN   c                   L    \ rS rSrSrSS\S   4S jjrS\4S jrS\4S	 jr	S
r
g)Retry`   z
Special exception to retry the job (if ``max_retries`` hasn't been reached).

:param defer: duration to wait before rerunning the job
Ndeferr6   c                 $    [        U5      U l        g N)r2   defer_score)selfr]   s     rO   __init__Retry.__init__g   s    */,rN   rR   c                 <    SU R                   =(       d    SS-  S S3$ )Nz<Retry defer r     0.2fzs>r`   ra   s    rO   __repr__Retry.__repr__j   s$     0 0 5A=dC2FFrN   c                     [        U 5      $ r_   )reprrh   s    rO   __str__Retry.__str__m   s    DzrN   rg   r_   )rD   rE   rF   rG   __doc__r   rb   rH   ri   rm   rM   rC   rN   rO   r[   r[   `   s3    7h'9: 7G# G rN   r[   c                   &    \ rS rSrS\S\4S jrSrg)JobExecutionFailedq   otherrR   c                 `    [        U[        5      (       a  U R                  UR                  :H  $ g)NF)rT   rq   args)ra   rs   s     rO   __eq__JobExecutionFailed.__eq__r   s&    e/0099

**rN   rC   N)rD   rE   rF   rG   r   rK   rv   rM   rC   rN   rO   rq   rq   q   s    C D rN   rq   c                   H    \ rS rSrS\S\\   4S jrS\4S jr	S\4S jr
Srg	)

FailedJobsx   countjob_resultsc                     Xl         X l        g r_   r{   r|   )ra   r{   r|   s      rO   rb   FailedJobs.__init__y   s    
&rN   rR   c                     U R                   S:X  a0  U R                  (       a  U R                  S   R                  nSU< 3$ U R                    S3SR                  S U R                   5       5      -   $ )Nr   r   z1 job failed z jobs failed:

c              3   L   #    U  H  n[        UR                  5      v   M     g 7fr_   )rl   result).0rs     rO   	<genexpr>%FailedJobs.__str__.<locals>.<genexpr>   s     =gVfQRd188nnVfs   "$)r{   r|   r   join)ra   excs     rO   rm   FailedJobs.__str__}   sd    ::?t//""1%,,C"3'**jj\1DII=gVZVfVf=g4gggrN   c                      S[        U 5       S3$ )N<>)rH   rh   s    rO   ri   FailedJobs.__repr__   s    3t9+QrN   r~   N)rD   rE   rF   rG   rL   r   r   rb   rH   rm   ri   rM   rC   rN   rO   ry   ry   x   s3    'c 'Y 'h h #  rN   ry   c                       \ rS rSrSrg)RetryJob   rC   N)rD   rE   rF   rG   rM   rC   rN   rO   r   r      s    rN   r   c            @           \ rS rSrSr SW\SSSSSSSSSSSSSS	SS
SSS	SSSSSSS\SSS.S\\\	S4      S\
\   S\
\\      S\
\   S\
\   S\S\
S   S\
S   S\
S   S\
S   S\
S   S\S\S\SSS SS!\S"SS#\
\   S$\S%SS&\
\   S'\
\\\4      S(\S)\S*\S+\
\   S,\
\   S-\S.\
\   S/\4>S0 jjjrSXS2 jrSXS3 jrSYS(\
\   S*\
\   S1\4S4 jjr\S1\4S5 j5       rSXS6 jrSXS7 jrSXS8 jrSXS9 jrS:\ \!   S1S4S; jr"S<\S=\S1S4S> jr#S<\S?\S@\
\!   SA\
\$   S!\SB\
\   SC\
\$   S1S4SD jr%S<\S@\
\!   S1S4SE jr&SXSF jr'SZSG\(SH\$SI\S1S4SJ jjr)SXSK jr*SL\+SM\,\+/S4   S1S4SN jr-S1\4SO jr.SXSP jr/SL\+S1S4SQ jr0SL\+S1S4SR jr1SL\+S1S4SS jr2SXST jr3S1\4SU jr4SVr5g)[Worker   a
  
Main class for running jobs.

:param functions: list of functions to register, can either be raw coroutine functions or the
  result of :func:`arq.worker.func`.
:param queue_name: queue name to get jobs from
:param cron_jobs:  list of cron jobs to run, use :func:`arq.cron.cron` to create them
:param redis_settings: settings for creating a redis connection
:param redis_pool: existing redis pool, generally None
:param burst: whether to stop the worker once all jobs have been run
:param on_startup: coroutine function to run at startup
:param on_shutdown: coroutine function to run at shutdown
:param on_job_start: coroutine function to run on job start
:param on_job_end: coroutine function to run on job end
:param after_job_end: coroutine function to run after job has ended and results have been recorded
:param handle_signals: default true, register signal handlers,
  set to false when running inside other async framework
:param job_completion_wait: time to wait before cancelling tasks after a signal.
  Useful together with ``terminationGracePeriodSeconds`` in kubernetes,
  when you want to make the pod complete jobs before shutting down.
  The worker will not pick new tasks while waiting for shut down.
:param max_jobs: maximum number of jobs to run at a time
:param job_timeout: default job timeout (max run time)
:param keep_result: default duration to keep job results for
:param keep_result_forever: whether to keep results forever
:param poll_delay: duration between polling the queue for new jobs
:param queue_read_limit: the maximum number of jobs to pull from the queue each time it's polled. By default it
                         equals ``max_jobs`` * 5, or 100; whichever is higher.
:param max_tries: default maximum number of times to retry a job
:param health_check_interval: how often to set the health check key
:param health_check_key: redis key under which health check is set
:param ctx: dictionary to hold extra user defined state
:param retry_jobs: whether to retry jobs on Retry or CancelledError or not
:param allow_abort_jobs: whether to abort jobs on a call to :func:`arq.jobs.Job.abort`
:param max_burst_jobs: the maximum number of jobs to process in burst mode (disabled with negative values)
:param job_serializer: a function that serializes Python objects to bytes, defaults to pickle.dumps
:param job_deserializer: a function that deserializes bytes into Python objects, defaults to pickle.loads
:param expires_extra_ms: the default length of time from when a job is expected to start
 after which the job expires, defaults to 1 day in ms.
:param timezone: timezone used for evaluation of cron schedules,
    defaults to system timezone
:param log_results: when set to true (default) results for successful jobs
  will be logged
NFTr   
   i,  i        ?   )
queue_name	cron_jobsredis_settings
redis_poolburst
on_startupon_shutdownon_job_start
on_job_endafter_job_endhandle_signalsjob_completion_waitmax_jobsjob_timeoutrP   rA   
poll_delayqueue_read_limitrB   health_check_intervalhealth_check_keyctx
retry_jobsallow_abort_jobsmax_burst_jobsjob_serializerjob_deserializerr&   r   log_results	functionsr8   r   r   r   r   r   r   r7   r   r   r   r   r   r   r   r   r6   rP   rA   r   r   rB   r   r   r   r   r   r   r   r   r&   r   r   c                R  ^  [        [        U5       V s0 s H  n U R                  U _M     sn T l        Uc  Ub  UR                  nO[        S5      eUT l        / T l        Ubs  [        S U 5       5      (       d  [        S5      e[        U5      T l        T R                  R                  T R                   V!s0 s H  n!U!R                  U!_M     sn!5        [        T R                  5      S:X  a  [        S5      eUT l        UT l        UT l        U	T l        U
T l        UT l        UT l        [(        R*                  " US-   5      T l        ST l        [1        U5      T l        [1        U5      T l        UT l        [1        U5      T l        U=(       d    [;        US-  S5      T l        ST l        UT l         [1        U5      T l!        Uc  T R                  [D        -   T l#        OUT l#        UT l$        T RH                  c  U=(       d
    [K        5       T l&        OS T l&        0 T l'        0 T l(        S T l)        [(        RT                  " 5       T l+        U=(       d    0 T l,        [;        U 4S	 jT R                  R[                  5        5       5      n"U"=(       d    SS
-   T l.        ST l/        ST l0        ST l1        ST l2        S T l3        UT l4        UT l5        T Rh                  (       a  T Rj                  (       aU  T Rm                  [n        Rp                  T Rr                  5        T Rm                  [n        Rt                  T Rr                  5        OTT Rm                  [n        Rp                  T Rv                  5        T Rm                  [n        Rt                  T Rv                  5        S T l<        UT l=        UT l>        ST l?        [        5       T lA        UT lB        UT lC        UT lD        UT lE        UT lF        Uc3  [        R                  " 5       R                  5       R                  T lK        g UT lK        g s  sn f s  sn!f )Nz4If queue_name is absent, redis_pool must be present.c              3   B   #    U  H  n[        U[        5      v   M     g 7fr_   )rT   r   )r   cjs     rO   r   "Worker.__init__.<locals>.<genexpr>   s     C2z"g..s   z'cron_jobs, must be instances of CronJobr   z4at least one function or cron_job must be registeredr   r   d   c              3   `   >#    U  H#  oR                   =(       d    TR                  v   M%     g 7fr_   )r?   job_timeout_s)r   fra   s     rO   r   r     s#     ]E\++;););;E\s   +.r   T)LmaprY   r=   r   r%   
ValueErrorr   r   allrW   listupdatelenr   r   r   r   r   r   r   rU   BoundedSemaphoresemjob_counterr3   r   r@   rA   poll_delay_smaxr   _queue_read_offsetrB   r   r'   r   _poolr    r   tasks	job_tasks	main_taskget_event_looploopr   valuesin_progress_timeout_sjobs_completejobs_retriedjobs_failed_last_health_check_last_health_check_log_handle_signals_job_completion_wait_add_signal_handlersignalSIGINThandle_sig_wait_for_completionSIGTERM
handle_sigon_stopr   r   allow_pick_jobssetaborting_tasksr   r   r   r&   r   r   now
astimezonetzinfor   )#ra   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rP   rA   r   r   rB   r   r   r   r   r   r   r   r   r&   r   r   r   r   max_timeouts#   `                                  rO   rb   Worker.__init__   sm   F SVVZ\eRf>gRfQqvvqyRf>g%'::
 !WXX$(* CCCC"#LMM!)_DNNN!!"H2277B;"HIt~~!#UVV
$&($* ++HqL9 !'4'4#6 &z2 0 JC1c4J"#"%/0E%F"#$(OO6M$MD!$4D!
::;I;\]_D"&D35
797;**,	9"]T^^EZEZE\]]&1&6Q"%<")*59#-$7!((((8[8[\((9\9\]((H((I<@$ 0%)(+,, 0 0& ?G>N113::T\_ ?h #Is   P7P$rR   c                    U R                   R                  U R                  5       5      U l         U R                   R	                  U R                  5        U R                   R	                  U R                  5       5        g! [
        R                   a     N@f = f! U R                   R	                  U R                  5       5        f = f)zE
Sync function to run the worker, finally closes worker connections.
N)r   create_taskmainr   run_until_completerU   CancelledErrorcloserh   s    rO   run
Worker.run.  s     ..tyy{;	7II((8
 II((6	 %% 		 II((6s#   %A? ?BB BB +Cc                    #    U R                   R                  U R                  5       5      U l        U R                  I Sh  vN   g N7f)zQ
Asynchronously run the worker, does not close connections. Useful when testing.
N)r   r   r   r   rh   s    rO   	async_runWorker.async_run;  s0      ..tyy{;nns   >A AAc                 ^  #    Ub  Xl         Ub  X l        U R                  5       I Sh  vN   U R                  (       aZ  U R                  R                  5       I Sh  vN  Vs/ s H  o3R                  (       a  M  UPM     nn[        U R                  U5      eU R                  $  N{ NJs  snf 7f)z
Run :func:`arq.worker.Worker.async_run`, check for failed jobs and raise :class:`arq.worker.FailedJobs`
if any jobs have failed.

:return: number of completed jobs
N)	r   r   r   r   poolall_job_resultssuccessry   r   )ra   r   r   r   failed_job_resultss        rO   	run_checkWorker.run_checkB  s      !(O%"0nn37993L3L3N-N-N!`-NV_V_!-N!`T--/ABB%%% 	-N!`s9   &B-B$2B-B&B-"B(:B( %B-&B-(B-c                 6    [        [        U R                  5      $ r_   )r   r   r   rh   s    rO   r   Worker.poolT  s    Hdjj))rN   c                 ~  #    U R                   cL  [        U R                  U R                  U R                  U R
                  U R                  S9I S h  vN U l         [        R                  S[        U R                  5      SR                  U R                  5      5        [        U R                  [        R                  5      I S h  vN   U R                  U R                  S'   U R                  (       a#  U R                  U R                  5      I S h  vN   [!        U R"                  5        S h  vN nU R%                  5       I S h  vN   U R&                  (       d  M4  SU R(                  s=::  a  U R+                  5       ::  a8  O  O5[,        R.                  " U R0                  R3                  5       6 I S h  vN     g U R                  R5                  U R
                  5      I S h  vN nUS:X  d  M  [,        R.                  " U R0                  R3                  5       6 I S h  vN     g  GN GNe GN N N N} NP N
 g 7f)N)r   r   r%   r&   z$Starting worker for %d functions: %sz, redisr   )r   r!   r   r   r   r   r&   loggerinfor   r   r   r"   r   r   r   r0   r   _poll_iterationr   r   _jobs_startedrU   gatherr   r   zcard)ra   _queued_jobss      rO   r   Worker.mainX  s    ::*##!%!6!6#22#'??!%!6!6  DJ 	:C<OQUQZQZ[_[i[iQjkTYY444 II??//$((+++D--. 
	 !&&(((zzz++Ct/A/A/CC!..$***;*;*=>>>$(IIOODOO$DD!#!..$***;*;*=>>>1 	5 ,
	 ( ?D> /s   AH=H(A3H=H+AH=H.H=-H;1H12H;5H=	H3
H=!AH=7H58.H=&H7'H=3.H=!H9"H=+H=.H=1H;3H=5H=7H=9H=;H=c                 4  #    U R                   nU R                  (       a?  U R                  S:  a/  U R                  U R                  5       -
  nUS:  a  g[	        X!5      nU R
                  (       a~  U R                  U R                  :  ad  [        5       nU R                  R                  U R                  [        S5      U R                  XS9I Sh  vN nU R                  U5      I Sh  vN   U R                  (       a  U R!                  5       I Sh  vN   [#        U R$                  R'                  5       5       H9  u  pVUR)                  5       (       d  M  U R$                  U	 UR+                  5         M;     U R-                  5       I Sh  vN   g N N N N7f)z
Get ids of pending jobs from the main queue sorted-set data structure and start those jobs, remove
any finished tasks from self.tasks.
r   r   Nz-inf)minstartnumr   )r   r   r   r   r  r   r   r   r1   r   zrangebyscorer   rJ   r   
start_jobsr   _cancel_aborted_jobsr   r   itemsdoner   
heart_beat)ra   r{   burst_jobs_remainingr   job_idsjob_idts          rO   r   Worker._poll_iterationt  sA    
 %%::$--2#'#6#69K9K9M#M #a',4E$--/"n $		 7 7OOvd>U>U[` !8 !  oog...  ++---djj..01IFvvxxJJv&
	 2 oo / . 	 sN   CFFF)F*(FF?F4F
FFFFFc                   #    U R                   R                  SS9 ISh  vN nUR                  [        SSS9  UR	                  [        [        5       [        -   [        S5      S9  UR                  5       I Sh  vN u  p#SSS5      ISh  vN   [        5       nW HD  nUR                  5       n U R                  U   nUR                  U5        UR                  5         MF     U(       aE  U R                  R!                  U5        U R                   R"                  " [        /UQ76 I Sh  vN   gg GN N N! , ISh  vN  (       d  f       N= f! [         a     M  f = f N87f)	zL
Go through job_ids in the abort_jobs_ss sorted set and cancel those tasks.
TtransactionNr   r   )r  endinf)r  r   )r   pipelinezranger$   zremrangebyscorer1   r#   rJ   executer   decoder   addcancelKeyErrorr   r   zrem)ra   pipeabort_job_idsr  abortedjob_id_bytesr  tasks           rO   r
  Worker._cancel_aborted_jobs  s%     99%%$%774KKQBK7!!-\^FW5W]bch]i!j%)\\^3M 87
  E)L!((*F~~f- F# * &&w/))..9999 ! 8  4 8777   :s   E&D2E&AD9:D5;D9E&D7#E&1E A+E&+E$,E&5D97E&9E?E EE&
E!E& E!!E&c                 `    U R                   S-
  U l         U R                  R                  5         g )Nr   )r   r   releaserh   s    rO   $_release_sem_dec_counter_on_complete+Worker._release_sem_dec_counter_on_complete  s%    ++a/rN   r  c           
        ^ #    U GH/  nT R                   R                  5       I Sh  vN   T R                  T R                  :  a  T R                   R	                  5           gT R                  S-   T l        UR                  5       n[        U-   nT R                  R                  SS9 ISh  vN nUR                  U5      I Sh  vN   UR                  U5      I Sh  vN nUR                  T R                  U5      I Sh  vN nU(       d  U(       a  U[        5       :  aX  T R                  S-
  T l        T R                   R	                  5         [        R                  SU5         SSS5      ISh  vN   GMu  UR!                  5         UR#                  U[%        T R&                  S-  5      S5         UR)                  5       I Sh  vN   T R*                  R-                  T R/                  U[%        U5      5      5      nUR1                  U 4S j5        UT R2                  U'   SSS5      ISh  vN   GM2     g GN GN GN| GNf GNE N N! [4        [6        4 aG    T R                  S-
  T l        T R                   R	                  5         [        R                  S	U5         Nf = f Nw! , ISh  vN  (       d  f       GM  = f7f)
zX
For each job id, get the job definition, check it's not running and start it in a task
Nr   Tr  z job %s already running elsewherere      1c                 $   > TR                  5       $ r_   )r)  )r  ra   s    rO   <lambda>#Worker.start_jobs.<locals>.<lambda>  s    $2[2[2]rN   z2multi-exec error, job %s already started elsewhere)r   acquirer   r   r(  r  r(   r   r  watchexistszscorer   r1   r   debugmultipsetexrL   r   r  r   r   run_jobadd_done_callbackr   r   r   )	ra   r  job_id_br  in_progress_keyr!  ongoing_existsscorer  s	   `        rO   r	  Worker.start_jobs  s      H((""$$$4==0  "#//!3D__&F4v=Oyy))d);;tjj111'+{{?'C!C"kk$//6BB!%,.2H (,'7'7!';D$HH$$&LL!CVL <;; 

OS1K1Kd1R-SUYZ
+,,.(( 		--dll63u:.NOA''(]^)*DJJv&3 <;;  $ <1!CB <  )%z2 _'+'7'7!';D$HH$$&LL!UW]^	_# <;;;s   $KH;BK,H>-K0J*IJ*I#J*IA%J*(K3I
4K;9J*5II	IAJ*%K0J(1K>KJ*J*J*
KIAJ%"J*$J%%J*(K*K	0J31K	=	Kr  r<  c                   ^ ^^^ ^!^"^#^$#    [        5       m$T R                  R                  SS9 IS h  vN nUR                  [        T-   5        UR                  [        T-   5        UR                  [        T-   S5        T R                  (       a3  UR                  [        T5        UR                  5       I S h  vN u  nm"pVOUR                  5       I S h  vN u  nm"nSnS S S 5      IS h  vN   Su  m!m Sm0 m#S[        SS 4UU U!UU"U#U U$4S	 jjnW(       d/  [        R                  S
T5        U" [        S5      5      I S h  vN $  [!        UT R"                  S9u  m!mm#nm W(       aK  [        5       T -
  S-  n
[        R)                  SU
TT!5        U" [*        R,                  " 5       5      I S h  vN $  T R.                  T!   n[3        US5      (       a	  T!n[4        nO	T ST! 3nS nU(       a=  UT":  a7  Um"T R                  R7                  [        T-   S[9        T"5      5      I S h  vN   UR:                  c  T R:                  OUR:                  nT"U:  a  [        5       T -
  S-  n
[        R                  SXU5        T =R<                  S-  sl        [?        T!TT#T"T S[        SU S35      T$[        5       UT R@                  TT RB                  S9n[*        RD                  " T RG                  TU5      5      I S h  vN $ [H        nS nSnURJ                  c  T RL                  OURJ                  nS nTT"[O        T 5      US.n0 T RP                  EUEnT RR                  (       a  T RS                  U5      I S h  vN   [        5       m$Sn [U        TT#5      nT"S:  a  ST" 3OSnT$U-
  S:  a  UST$U-
  S-  S S 3-  n[        R)                  S!T$T -
  S-  UUU5        T RV                  RY                  URZ                  " U/TQ70 T#D65      =T R\                  T'   n [*        R^                  " UU5      I S h  vN nUb  T R`                  (       d  SO[c        [e        U5      5      n T R\                  T	  Sn[        5       n[        R)                  S#UT$-
  S-  UU5        SnT =Rl                  S-  sl6        UR                  c  T R                  OUR                  nUR                  c  T R                  OUR                  nS nU[H        La6  U(       d  US%:  a)  [?        T!TT#T"T UUT$UUT R@                  TT RB                  S9nT R                  (       a  T R                  U5      I S h  vN   [*        RD                  " T R                  TUUUUUU5      5      I S h  vN   T R                  (       a  T R                  U5      I S h  vN   g g  GN GN GN GN! , IS h  vN  (       d  f       GN= f GN! [$         a1  n	[        R'                  ST5        U" U	5      I S h  vN  s S n	A	$ S n	A	ff = f GN|! [0         a9    [        R                  STT!5        U" [        ST!< S35      5      I S h  vN  s $ f = f GNN GN~ GN GNM! [f        [*        R,                  4 a*  n	[i        U	S"S 5      n[k        U5      (       a  U" 5       ne S n	A	ff = f! T R\                  T	 f = f! [f        [*        R,                  4 Ga  n	[        5       nUT$-
  S-  n
T Rn                  (       a  [q        U	[r        5      (       a  U	Rt                  n[        R)                  S$XU	Rt                  =(       d    S%S-  5        U	Rt                  (       a  U	Rt                  [        5       U-
  -   nT =Rv                  S-  sl;         S n	A	GNTT Rx                  ;   ao  [q        U	[*        R,                  5      (       aP  [        R)                  S&X5        U	nSnT Rx                  R{                  T5        T =R<                  S-  sl         S n	A	GN{T Rn                  (       aV  [q        U	[*        R,                  [|        45      (       a1  [        R)                  S'X5        T =Rv                  S-  sl;         S n	A	GN[        R'                  S(XU	R~                  R                  U	S"U0S)9  U	nSnT =R<                  S-  sl         S n	A	GN.S n	A	ff = f GN GNV GN/7f)*NTr  iPY F)z	<unknown>r   rC   r   rR   c                   >#    T=R                   S-  sl         [        TTTTTSU T	[        5       T ST 3TR                  TR                  TS9n[
        R                  " TR                  TU5      5      I S h  vN   g  N7f)Nr   F:)functionru   kwargsjob_tryenqueue_time_msr   r   start_msfinished_msref
serializerr   r  )r   r   r1   r   r   rU   shieldfinish_failed_job)
r   result_data_ru   rD  function_namer  rC  rB  ra   rE  s
     rO   
job_failed"Worker.run_job.<locals>.job_failed  s     !+& /!(Nha/..??L ..!7!7!MNNNs   A5B 8A>9B zjob %s expiredzjob expired)deserializerzdeserializing job %s failedre   u%   %6.2fs ⊘ %s:%s aborted before startzjob %s, function %r not foundz	function z
 not foundnext_runr@  z#%6.2fs ! %s max retries %d exceededr   zmax z retries exceeded)r  rH  )r  rC  enqueue_timer<  z try= i  z	 delayed=rf   su   %6.2fs → %s(%s)%sextrau   %6.2fs ← %s ● %su$   %6.2fs ↻ %s retrying job in %0.2fsr   u   %6.2fs ⊘ %s abortedu*   %6.2fs ↻ %s cancelled, will be run againz%6.2fs ! %s failed, %s: %s)rT  )Fr1   r   r  getr)   incrr,   expirer   r   r$   r  BaseExceptionr   warningrq   r   r   r   	exceptionr   rU   r   r   r  hasattrr*   setexrH   rB   r   r   r   r   rI  rJ  	no_resultr?   r   r/   r   r   r-   r   r   r>   r   wait_forr   r5   rl   	Exceptiongetattrcallabler   r   rT   r[   r`   r   r   remover   	__class__rD   rA   r@   r   
finish_jobr   )%ra   r  r<  r!  vr  	abort_jobrM  enqueue_job_tryer  rA  rG  keep_in_progressrB   result_datar   	exc_extrafinishr?   
incr_scorejob_ctxr   r   rS  rT  r%  
result_strrF  rA   result_timeout_sru   rD  rL  rC  rB  rE  s%   ``                             @@@@@@rO   r7  Worker.run_job  s    >99%%$%774HH^f,-II&/0KK(6159$$		-015+?(7Ay&*lln 47A!	 87 *8& "!#	O- 	OD 	O 	O& NN+V4#$6}$EFFF	'L_ 5 5MIM4/ /1T9AKK?FMZ#G$:$:$<===	a151NH
 8Z((C0EHAm_-C#8%G))//"2V";UCLQQQ&.&8&8&@DNNhFXFX	Y/1T9ANN@!)T!*"T)4E#FG..K !(>(>v{(STTT	*2*<*<*DD&&(J\J\	$(
*?;	
 &%W%##C(((>3	$tV,A)01eG9%"E5 D(9h&6$%>t$DAFFKK-?0Jd/RTWYZ\ab,0II,A,A(BTBTUXBj[_BjciBj,kkDNN6"T
+&//i@@ $*>9I9IRxX\]cXdOe
NN6*8 G&.KKK.x1G40OQTV`aF!# )1(D(D(LD$$RZRnRn 	 2:1G1G1O4--U]UkUk"(;?ORS?S*..K  ??//#&&&nnOO # 

 
	
 
	
 $$S))) O 8 ,@ 4 8777H G " 	':FC#A&&&	' >  	aNN:FMR#$6=BSS]7^$_````	a R, U  ) Aw556 #Aw5	I&& )I	 NN6*7112 	&&.Kx'4/A:a#7#7]]
BAQ]]M_^_cgLgh==!",.52H!IJ!!Q&!!4...:aAWAW3X3X3Q<##**62  A%  ZG4J4JH3U%V%VH!Q!!Q&!!  0!!++:N:NPQZaclYm !    A%  1	&n '
	
 *s  (c/Wc/BW:W;WW

W"c/-W.Ac/W+c/	W. #Ac/0X,1c/6X/ A"c/'Y5(Cc/:Y8;A:c/5Y;6c/B[ Z 1Y>2Z 6*[ ![ /C1c/ c&!2c/c))c/=c,>c/W
Wc/W(WW(#	c/.
X)8!X$XX$X)c/$X))c//:Y2)Y,*Y2/c/1Y22c/8c/;c/>Z [%[  [[ [[ c#5B1c&c/,A9c%c/+A!cc/Acc/c##c/)c/,c/rl  rj  rp  rm  ri  c                   #    U R                   R                  SS9 IS h  vN n/ n	[        U-   n
Uc  X/-  n	OUR                  U
[	        U5      5        U(       a{  U(       a,  U(       a  S OUnUR                  [        U-   U[	        U5      S9  U	[        U-   [        U-   /-  n	UR                  [        U5        UR                  U R                  U5        O#U(       a  UR                  U R                  Xa5        U	(       a  UR                  " U	6   UR                  5       I S h  vN   S S S 5      IS h  vN   g  GN N N
! , IS h  vN  (       d  f       g = f7f)NTr  px)r   r  r(   pexpirer2   r   r+   r,   r)   r   r$   r   zincrbydeleter  )ra   r  rl  rj  rp  rA   rm  ri  trdelete_keysr:  rW  s               rO   rd  Worker.finish_job  s     99%%$%772K4v=O'00

?E2B,CD%8T>NFFF,v5{uV}FU 06 9>F;RSSv.0

4??J?		;'**,' 877& ' 8777sW   ED8EC>D?"D;#D?'E2D=3E;D?=E?EEEEc                 l  #    U R                   R                  SS9 IS h  vN nUR                  [        U-   [        U-   [
        U-   5        UR                  [        U5        UR                  U R                  U5        U R                  =(       d    U R                  S:  nUbG  U(       a@  U R                  (       a  SOU R                  nUR                  [        U-   U[        U5      S9  UR                  5       I S h  vN   S S S 5      IS h  vN   g  N N N	! , IS h  vN  (       d  f       g = f7f)NTr  r   rs  )r   r  rw  r,   r(   r)   r   r$   r   rA   r@   r   r+   r2   r  )ra   r  rj  rx  rP   rW  s         rO   rJ  Worker.finish_failed_job  s     99%%$%772II 6)&/'
 GGM6*GGDOOV,22Ld6H6H16LK&;"66D<N<N(61;5=Q**, 877  8777sW   D4DD4CD>D?DD4DD4DD4D1 D#!D1-D4c                    #    [         R                  " U R                  S9nU R                  5       I S h  vN   [	        U R
                  S5      nU R                  X5      I S h  vN   g  N4 N7f)Ntzr   )r   r   r   record_healthr   r   run_cron)ra   r   cron_window_sizes      rO   r  Worker.heart_beat  sX     lldmm,  """t00#6mmC222 	# 	3s!   2A-A).A-#A+$A-+A-ndelaynum_windowsc                   #    [        5       n[        X#-  S9nX-   nU R                   GH5  nUR                  c+  UR                  (       a  Xl        OUR                  U5        M<  UR                  U:  d  MN  UR                  (       a  UR                  nO7UR                  (       a$  UR                   S[        UR                  5       3OS nUR                  U R                  R                  UR                  UU R                  UR                  [        R                  " U R                   S9:  a  UR                  OS S95        UR                  UR                  5        GM8     U=(       a    ["        R$                  " U6 I S h  vN   g   g  N7f)N)secondsr@  r~  )_job_id_queue_name_defer_until)r   r   r   rP  run_at_startupcalculate_nextr  uniquer=   r4   r  r   enqueue_jobr   r   r   r   rU   r   )	ra   r  r  r  job_futures
cron_delaythis_hb_cutoffcron_jobr  s	            rO   r  Worker.run_cron  s7    eu':;
H  (**()%++A.   >1??,4OOFS[SbSba
8;L;L0M/NOhlFII))  &$(OO191B1BX\\UYUbUbEc1cH--im * 	 ''(9(9:5 '8 	:gnnk:::::s   A0F6DFF	Fc                 $  #    [        5       nXR                  -
  U R                  :  a  g Xl        [        S U R                  R                  5        5       5      nU R                  R                  U R                  5      I S h  vN n[        R                  " 5       S SU R                   SU R                   SU R                   SU SU 3nU R                  R                  U R                  [!        U R                  S-   S	-  5      UR#                  5       5      I S h  vN   XDR%                  S
5      S  nU R&                  (       a,  XPR&                  :w  a  [(        R+                  SU5        XPl        g U R&                  (       d  XPl        g g  GN	 Np7f)Nc              3   J   #    U  H  oR                  5       (       + v   M     g 7fr_   r  r   r  s     rO   r   'Worker.record_health.<locals>.<genexpr>  s     F2EQLL2E   !#z%b-%d %H:%M:%Sz j_complete=
 j_failed= j_retried= j_ongoing=z queued=r   re   zj_complete=zrecording health: %s)r	   r   r   sumr   r   r   r   r   r   r   r   r   r   r6  r   rL   encodeindexr   r   r   )ra   now_tspending_tasksqueuedr   
log_suffixs         rO   r  Worker.record_health   s[    ,,,0J0JJ"(F$**2C2C2EFFyyt77||~n-\$:L:L9MZX\XhXhWi j**+;}oXfXW 	 ii!!3(B(BQ(F$'N#OQUQ\Q\Q^
 	
 	
 **]356
&&:9T9T+TKK.5*4',,*4' - 8
	
s&   B FFBFFA.FFsignumhandlerc                      U R                   R                  U[        X!5      5        g ! [         a    [        R                  S5         g f = f)Nz@Windows does not support adding a signal handler to an eventloop)r   add_signal_handlerr   NotImplementedErrorr   r4  )ra   r  r  s      rO   r   Worker._add_signal_handler  s?    	]II((1IJ" 	]LL[\	]s   %( A
	A
c                 z    U R                   U R                  -   U R                  -   [        U R                  5      -   $ r_   )r   r   r   r   r   rh   s    rO   r   Worker._jobs_started  s1    !!D$5$558H8HH3tzz?ZZrN   c                    #    [        U R                  5      (       a;  [        R                  " S5      I Sh  vN   [        U R                  5      (       a  M:  gg N"7f)zH
Sleeps until all tasks are done. Used together with asyncio.wait_for()
g?N)r   r   rU   sleeprh   s    rO   _sleep_until_tasks_complete"Worker._sleep_until_tasks_complete  s8      $**oo--$$$ $**oo$s   4AAAAc                   #    [         R                  " [        R                  5         [        R                  " U R                  5       U R                  5      I Sh  vN   SSS5        [        R                  SUR                  U R                  U R                  U R                  [        S U R                  R                  5        5       5      5        U R                  R                  5        H*  nUR!                  5       (       a  M  UR#                  5         M,     U R$                  =(       a    U R$                  R#                  5         U R&                  =(       a    U R'                  U5        g  g GN! , (       d  f       GN= f7f)z`
Wait for tasks to complete, until `wait_for_job_completion_on_signal_second` has been reached.
Nuh   shutdown on %s, wait complete ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d ongoing to cancelc              3   J   #    U  H  oR                  5       (       + v   M     g 7fr_   r  r  s     rO   r   5Worker._wait_for_tasks_to_complete.<locals>.<genexpr>4  s     :&9FFH&9r  )
contextlibsuppressrU   TimeoutErrorr^  r  r   r   r   r=   r   r   r   r  r   r   r  r  r   r   )ra   r  r  s      rO   _wait_for_tasks_to_complete"Worker._wait_for_tasks_to_complete%  s	       !5!56""002))   7
 	vKK:djj&7&7&9::	
 ""$A6688
 % 	24>>002-f--! 76s6   %F3E0E-E0B"FA(F-E00
E?:Fc           
      L   [        U5      n[        R                  S5        SU l        [        R                  SUR                  U R
                  U R                  U R                  [        U R                  5      5        U R                  R                  U R                  US95        g)z
Alternative signal handler that allow tasks to complete within a given time before shutting down the worker.
Time can be configured using `wait_for_job_completion_on_signal_second`.
The worker will stop picking jobs when signal has been received.
z"Setting allow_pick_jobs to `False`FuW   shutdown on %s ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d to be completed)r  N)r   r   r   r   r=   r   r   r   r   r   r   r   r  )ra   r  sigs      rO   r   %Worker.handle_sig_wait_for_completion<  s     fo89$eHH

O	
 			d>>c>JKrN   c           
         [        U5      n[        R                  SUR                  U R                  U R
                  U R                  [        U R                  5      5        U R                  R                  5        H*  nUR                  5       (       a  M  UR                  5         M,     U R                  =(       a    U R                  R                  5         U R                  =(       a    U R                  U5        g   g )NuY   shutdown on %s ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d ongoing to cancel)r   r   r   r=   r   r   r   r   r   r   r  r  r   r   )ra   r  r  r  s       rO   r   Worker.handle_sigO  s    fogHH

O	
 ""$A6688
 % 	24>>002*c**rN   c                   #    U R                   (       d  U R                  [        R                  5        U R                  (       d  g [
        R                  " U R                  R                  5       6 I S h  vN   U R                  R                  U R                  5      I S h  vN   U R                  (       a#  U R                  U R                  5      I S h  vN   U R                  R                  SS9I S h  vN   S U l        g  N Nc N1 N7f)NTclose_connection_pool)r   r   r   SIGUSR1r   rU   r   r   r   r   rw  r   r   r   r   rh   s    rO   r   Worker.close_  s     ##OOFNN+zznndjj//1222iit44555""488,,,iiooDo999
 	35,9sH   A1D
3D4,D
 D!3D
D D
5D6D
D
D
D
c                     SU R                    SU R                   SU R                   S[        S U R                  R                  5        5       5       S3	$ )Nz<Worker j_complete=r  r  r  c              3   J   #    U  H  oR                  5       (       + v   M     g 7fr_   r  r  s     rO   r   "Worker.__repr__.<locals>.<genexpr>n  s     G3Fa\\3Fr  r   )r   r   r   r  r   r   rh   s    rO   ri   Worker.__repr__k  s]    !$"4"4!5Z@P@P?QQ\]a]n]n\o pG4::3D3D3FGGHK	
rN   )/r   r   r   r   r   r   r   r   r   r   r   r   r   r&   r   r   r   r   r   r   r   r   r   r   r   r   rA   r@   r   r   r   r   r   rB   r   r   r   r   r   r   r   r   r   r   r   r   r   )rC   )rR   NNN)   )6rD   rE   rF   rG   ro   r%   r&   r   r   r;   r   rH   r   r    r   rK   rL   r   r   r   r   r   rb   r   r   r   propertyr   r   r   r
  r)  r   bytesr	  r7  rJ   rd  rJ  r  r   r  r  r   r   r   r   r  r  r   r   r   ri   rM   rC   rN   rO   r   r      s+   +^ CEr] %71526)-2637482659##$*-*.$)),*.48*.(,!& /337 0'+ Cr]E(,="=>?r] SM	r]
 HW-.r] !/r] X&r] r] ./r] /0r] 01r] ./r]   12r] r] !r]  !r]" (#r]$ (%r]& "'r]( ')r]* #3-+r], -r].  2/r]0 #3-1r]2 d38n%3r]4 5r]6 7r]8 9r]: !,;r]< #<0=r]> ?r]@ 8$Ar]B Cr]h7&(4. &QYZ]Q^ &jm &$ *h * * 8 >:0(+U (+ (+TJ*C J* J* J*X  e_	
 #5/ " SM #5/ 
>c  TX  3#; #; #;S #;QU #;J5*]' ]HgYPT_<U ]Z^ ][s [%. .D ..LW L L&+ +T + 

# 
rN   r   settings_clsr9   c                 "   [        [        R                  " [        5      R                  R                  5       5      n[        U [        5      (       a  U OU R                  nUR                  5        VVs0 s H  u  p4X1;   d  M  X4_M     snn$ s  snnf r_   )
r   inspect	signaturer   
parameterskeysrT   dict__dict__r  )r  worker_argsdkre  s        rO   
get_kwargsr  r  sh    g''/::??ABK"<66L<Q<QAWWY;YTQ!*:DADY;;;s   2BBrB  c                 2    [        S0 0 [        U 5      EUED6$ )NrC   )r   r  )r  rB  s     rO   create_workerr  x  s     ;:z,/:6:;;rN   c                 >    [        U 40 UD6nUR                  5         U$ r_   )r  r   )r  rB  workers      rO   
run_workerr  |  s    <262F
JJLMrN   r   r   r   c                 v  #    U =(       d
    [        5       n [        U 5      I S h  vN nU=(       d    [        nU=(       d	    U[        -   nUR	                  U5      I S h  vN nU(       d  [
        R                  S5        SnO[
        R                  SU5        SnUR                  SS9I S h  vN   U$  N NV N
7f)Nz9Health check failed: no health check sentinel value foundr   zHealth check successful: %sr   Tr  )	r    r!   r%   r'   rU  r   rY  r   r   )r   r   r   r   datar   s         rO   async_check_healthr    s      $6}N'77E11J'QJ9P,P+,,DRS148
++D+
111H 8 - 2s4   "B9B39B9B5AB9,B7-B95B97B9c                 <   [        U 5      n[        [        [           UR	                  S5      5      n[        [        [
           UR	                  S5      5      n[        [        [
           UR	                  S5      5      n[        R                  " [        X#U5      5      $ )zk
Run a health check on the worker and return the appropriate exit code.
:return: 0 if successful, 1 if not
r   r   r   )	r  r   r   r    rU  rH   rU   r   r  )r  
cls_kwargsr   r   r   s        rO   check_healthr    st    
 L)J(=1:>>BR3STNHSM:>>:L+MNhsmZ^^L%ABJ;;).JWXXrN   r  )YrU   r  r  loggingr   dataclassesr   r   r   r   	functoolsr   r   r	   typingr
   r   r   r   r   r   r   r   r   r   r   redis.exceptionsr   r   arq.cronr   arq.jobsr   r   r   r   r   r   connectionsr   r    r!   r"   	constantsr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   utilsr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   	getLoggerr   objectr]  r;   rH   rK   rL   rY   rW   r[   rq   ry   r   r   	NameErrorr  r  r  r  r  rC   rN   rO   <module>r     s        ! 2 2    h h h h 6  s s M M  
 
 
 ^^			<	(H	    04,0*.#!wS($556!w 3-!w ,-	!w
 ()!w "$!w }!w !wHL "     	| 	c
 c
L<1 <d3	>6J << 4 < < <1 S V  rv]+?G}aijman&	Y3 	Y 	YrN   