o
    1 iK&                  	   @   s6  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& e'e(Z)dZ*dZ+e&G dd de"Z,e&	dde
e	ge	f dee- ded fddZ.dS )    N)partial)Number)AnyCallableDictOptionalType)RunnerThreadStartTraceback)_ERROR_FETCH_TIMEOUT)_TrainingResult)	TrialInfo_TrainSessionget_sessioninit_sessionshutdown_session)PlacementGroupFactory)DEFAULT_METRICRESULT_DUPLICATESHOULD_CHECKPOINT)	Trainable)_detect_config_single)DeveloperAPIz.null_markerz.temp_markerc                   @   s   e Zd ZdZdZdd Zdeeef fddZ	dd	 Z
d
d Zdd ZddefddZdefddZdd Zdd ZdddZdS )FunctionTrainablezwTrainable that runs a user function reporting results.

    This mode of execution does not support checkpoint/restore.funcc                    sR   t  fddt j j j jjd d  jjd jdd d d d d d d d d  _d S )Nc                            jS N_trainable_funcconfig selfr    q/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/ray/tune/trainable/function_trainable.py<lambda>,       z)FunctionTrainable.setup.<locals>.<lambda>nameid	resourcesZlogdirZ	driver_ipZdriver_node_idZexperiment_nameT)training_func
trial_infostorageZsynchronous_result_reportingZ
world_rankZ
local_rankZ	node_rankZlocal_world_sizeZ
world_sizeZdataset_shard
checkpoint)	r   r   
trial_nametrial_idtrial_resources_storageZtrial_driver_staging_pathexperiment_dir_name_last_training_resultr"   r   r    r!   r#   setup*   s,   
	
zFunctionTrainable.setupr   c                 C   s   t )z7Subclasses can override this to set the trainable func.)NotImplementedErrorr4   r    r    r#   r   C   s   z!FunctionTrainable._trainable_funcc                    sN    fdd}t | jdd _ j  z j  W d S  ty&   Y d S w )Nc               
      s.   z   jW S  ty }  zt| d } ~ ww r   )r   r   	Exceptionr
   )er!   r    r#   
entrypointI   s   z,FunctionTrainable._start.<locals>.entrypointT)targetZerror_queuedaemon)r	   _error_queueZ_runnerZ_status_reporter_startstartRuntimeError)r"   r9   r    r!   r#   r=   H   s   
zFunctionTrainable._startc                 C   sZ   t  }|js
|  | }|std|j}t|v rd|t< || _|j	dur+d|t< |S )a=  Implements train() for a Function API.

        If the RunnerThread finishes without reporting "done",
        Tune will automatically provide a magic keyword __duplicate__
        along with a result with "done=True". The TrialRunner will handle the
        result accordingly (see tune/tune_controller.py).
        zShould not have reached here. The TuneController should not have scheduled another `train` remote call.It should have scheduled a `stop` instead after the training function exits.FNT)
r   Ztraining_startedr>   Zget_nextr?   metricsr   r   r3   r-   )r"   sessionZtraining_resultr@   r    r    r#   step\   s   
zFunctionTrainable.stepc                 C   s   || S r   r    )r"   fnr    r    r#   execute   s   zFunctionTrainable.execute checkpoint_dirc                 C   s   |rt d| jS )Nz4Checkpoint dir should not be used with function API.)
ValueErrorr3   )r"   rF   r    r    r#   save_checkpoint   s   z!FunctionTrainable.save_checkpointcheckpoint_resultc                 C   s   t  }|j|_d S r   )r   r-   Zloaded_checkpoint)r"   rI   rA   r    r    r#   load_checkpoint   s   z!FunctionTrainable.load_checkpointc                 C   s8   t  }z|jdd W |  t  d S |  t  w )Nr   timeout)r   finish_report_thread_runner_errorr   )r"   rA   r    r    r#   cleanup   s   
zFunctionTrainable.cleanupc                    sv   t  }ttjdd}|j|d |j rdS |j fddt	 j
 j j jjd d  jjd jd i  _d	S )
NZTUNE_FUNCTION_THREAD_TIMEOUT_S   rK   Fc                      r   r   r   r    r!   r    r#   r$      r%   z0FunctionTrainable.reset_config.<locals>.<lambda>r&   )r*   r+   r,   T)r   intosenvirongetrM   Ztraining_threadis_aliveresetr   r.   r/   r0   r1   Ztrial_working_directoryr2   Z_last_result)r"   Z
new_configrA   Zthread_timeoutr    r!   r#   reset_config   s(   

	zFunctionTrainable.reset_configFc                 C   s.   z| j j|td}t| tjy   Y d S w )N)blockrL   )r<   rT   r   r
   queueEmpty)r"   rX   r8   r    r    r#   rN      s   z-FunctionTrainable._report_thread_runner_errorN)rE   )F)__name__
__module____qualname____doc___namer5   r   strr   r   r=   rB   rD   rH   r   rJ   rO   rW   rN   r    r    r    r#   r   "   s    *	r   
train_funcr'   returnc                    sn   t f}tdrj| }tj}t}|s td|t	dd G  fdddg|R  }|S )N
__mixins__zUnknown argument found in the Trainable function. The function args must include a single 'config' positional parameter.
Found: {}Z
_resourcesc                       s^   e Zd Z pedrj ndZdd ZfddZedee	e
f dee ffd	d
ZdS )z#wrap_function.<locals>.ImplicitFuncr[   r   c                 S   s   | j S r   )r_   r!   r    r    r#   __repr__   s   z,wrap_function.<locals>.ImplicitFunc.__repr__c                    sX   t  |}dd }d }t r| D ]}|| qn| }|| t tdi |S )Nc                 S   sH   | sd S t | trt |  d S t | tr t t| i d S td)NzuInvalid return or yield value. Either return/yield a single number or a dictionary object in your trainable function.)
isinstancedictr   reportr   r   rG   )outputr    r    r#   handle_output   s   

zJwrap_function.<locals>.ImplicitFunc._trainable_func.<locals>.handle_outputT)r   inspectisgeneratorfunctionr   rg   r   )r"   r   rC   ri   rh   )ra   r    r#   r      s   



z3wrap_function.<locals>.ImplicitFunc._trainable_funcr   rb   c                    s   t  tst r |S  S r   )re   r   callable)clsr   )r)   r    r#   default_resource_request  s   z<wrap_function.<locals>.ImplicitFunc.default_resource_requestN)r[   r\   r]   hasattrr_   rd   r   classmethodr   r`   r   r   r   rn   r    r'   r)   ra   r    r#   ImplicitFunc   s    
rr   )
r   ro   rc   rj   getfullargspecargsr   rG   formatgetattr)ra   r'   Zinherit_fromZ	func_argsZuse_config_singlerr   r    rq   r#   wrap_function   s   

/rw   r   )/rj   loggingrR   rY   	functoolsr   numbersr   typingr   r   r   r   r   Zray.air._internal.utilr	   r
   Zray.air.constantsr   Z&ray.train._internal.checkpoint_managerr   Zray.train._internal.sessionr   r   r   r   r   Z#ray.tune.execution.placement_groupsr   Zray.tune.resultr   r   r   Zray.tune.trainable.trainabler   Zray.tune.utilsr   Zray.util.annotationsr   	getLoggerr[   loggerZNULL_MARKERZTEMP_MARKERr   r`   rw   r    r    r    r#   <module>   s>    
 $