o
    ü* iÛ  ã                   @   sJ   d dl Z d dlZd dlmZ d dlZddlmZmZmZ G dd„ dƒZ	dS )é    N)ÚThreadé   )Úget_gpu_infoÚget_gpu_processÚget_gpu_utilc                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚWatcherc                 C   s¾   || _ d| _g | _| j jjsd S t ¡ rd S | j jjp | j jj	j
| _t| jƒdkrKtj | j jj| j jj› d¡}tjtj |¡dd t|dƒ| _nd S t| jd| _d| j_| j ¡  d S )Né   r   z.gpu.logT)Úexist_okÚw)Útarget)ÚctxÚintervalZgpu_utilÚargsZenable_gpu_logÚpaddleZis_compiled_with_rocmZdevicesÚnodeZdeviceÚlabelsÚgpusÚlenÚosÚpathÚjoinZlog_dirZjob_idÚmakedirsÚdirnameÚopenÚgpu_fdr   ÚwatchÚprocÚdaemonÚstart)Úselfr   Úfn© r!   úy/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/paddle/distributed/launch/controllers/watcher.pyÚ__init__   s$   
ÿzWatcher.__init__c                 C   s€   t | jƒdks	d S |  ¡  d}| j |¡ | j d¡ | jj ¡ s2|  |¡ t	 
| j¡ | jj ¡ r!t| dƒr>| j ¡  d S d S )Nr   zDindex,utilization_gpu,memory_total,memory_used,memory_free,timestampÚ
r   )r   r   Ú_print_gpu_infor   Úwriter   ÚstatusZis_doneÚ_save_gpu_logÚtimeÚsleepr   ÚhasattrÚclose)r   Úutil_keyr!   r!   r"   r   6   s   
þ
ÿzWatcher.watchc              
   C   sø   z^d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q| j  d¡ d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q?| j  d¡ | j  ¡  W d S  ty{ } z| jj	 
d|›¡ W Y d }~d S d }~ww )NzEindex,uuid,driver_version,name,gpu_serial,display_active,display_moder$   z.pid,process_name,gpu_uuid,gpu_name,used_memoryzsave gpu info failed: )r   r&   r   r   Ústrr   ÚflushÚ	Exceptionr   ÚloggerÚwarning)r   Zinfo_keyÚlineZprocess_keyÚer!   r!   r"   r%   G   s(   "€ÿzWatcher._print_gpu_infoc              
   C   sz   zt | jƒD ]}| j | |¡¡ | j d¡ q| j ¡  W d S  ty< } z| jj 	d|›¡ W Y d }~d S d }~ww )Nr$   zsave gpu log failed: )
r   r   r   r&   r.   r/   r0   r   r1   r2   )r   r-   r3   r4   r!   r!   r"   r(   ]   s   "€ÿzWatcher._save_gpu_logc                 C   s   t | dƒr	 d S d S )Nr   )r+   )r   r!   r!   r"   Ústopf   s   
ýzWatcher.stopN)Ú__name__Ú
__module__Ú__qualname__r#   r   r%   r(   r5   r!   r!   r!   r"   r      s    	r   )
r   r)   Ú	threadingr   r   Zutils.nvsmir   r   r   r   r!   r!   r!   r"   Ú<module>   s   