o
    pi8                     @  sF  d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	Z
d dlZddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ erLd dlmZ G dd dZG dd dZG dd dZG dd dZdad'ddZG dd dZG dd dZ dZ!dZ"dZ#dZ$dZ%d Z&e
'e
j(j)d! d Z*d"Z+d(d#d$Z,	 d)d%d&Z-dS )*    )annotationsN)TYPE_CHECKINGAny   )core   )MP_STATUS_CHECK_INTERVALCleanupFuncRegistrar_cleanup_mmap   )_IterableDatasetFetcher_MapDatasetFetcher)_flatten_batch)Datasetc                   @  s   e Zd Zdd ZdS )_IterableDatasetStopIterationc                 C  s
   || _ d S N)	worker_id)selfr    r   b/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddle/io/dataloader/worker.py__init__(   s   
z&_IterableDatasetStopIteration.__init__N)__name__
__module____qualname__r   r   r   r   r   r   '   s    r   c                   @  s   e Zd ZdS )_ResumeIterationN)r   r   r   r   r   r   r   r   ,   s    r   c                   @  s    e Zd ZdZdZedd ZdS )_DatasetKindr   r   c                 C  s>   | t jkrt||||S | t jkrt||||S td|  )Nzunknown Dataset kind )r   MAPr   ITERr   NotImplementedError)kinddatasetauto_collate_batch
collate_fn	drop_lastr   r   r   create_fetcher4   s   

z_DatasetKind.create_fetcherN)r   r   r   r   r   staticmethodr$   r   r   r   r   r   0   s
    r   c                   @  s   e Zd Zdd Zdd ZdS )ParentWatchDogc                 C  s   t  | _d| _d S NT)osgetppid_parent_pid_parent_aliver   r   r   r   r   E   s   

zParentWatchDog.__init__c                 C  s   | j rt | jk| _ | j S r   )r+   r(   r)   r*   r,   r   r   r   is_aliveI   s   zParentWatchDog.is_aliveN)r   r   r   r   r-   r   r   r   r   r&   D   s    r&   return
WorkerInfoc                   C  s   t S )a!  
    Get DataLoader worker process information function, this function is
    used to split data copy in worker process for IterableDataset
    (see :code:`paddle.io.IterableDataset`), worker information contains
    following fields:

    :attr:`num_workers`: total worker process number, see `paddle.io.DataLoader`

    :attr:`id`: the worker process id, count from 0 to :attr:`num_workers - 1`

    :attr:`dataset`: the dataset object in this worker process

    Returns:
        WorkerInfo: an instance of WorkerInfo which contains fields above.

    Notes:
        For more usage and examples, please see :code:`paddle.io.IterableDataset`

    Example:

        .. code-block:: python

            >>> import math
            >>> import paddle
            >>> import numpy as np
            >>> from paddle.io import IterableDataset, DataLoader, get_worker_info

            >>> class SplitedIterableDataset(IterableDataset): # type: ignore[type-arg]
            ...     def __init__(self, start, end):
            ...         self.start = start
            ...         self.end = end
            ...
            ...     def __iter__(self):
            ...         worker_info = get_worker_info()
            ...         if worker_info is None:
            ...             iter_start = self.start
            ...             iter_end = self.end
            ...         else:
            ...             per_worker = int(
            ...                 math.ceil((self.end - self.start) / float(
            ...                     worker_info.num_workers)))
            ...             worker_id = worker_info.id
            ...             iter_start = self.start + worker_id * per_worker
            ...             iter_end = min(iter_start + per_worker, self.end)
            ...
            ...         for i in range(iter_start, iter_end):
            ...             yield np.array([i])
            ...
            >>> place = paddle.CPUPlace()
            >>> dataset = SplitedIterableDataset(start=2, end=9)
            >>> dataloader = DataLoader(
            ...     dataset,
            ...     places=place,
            ...     num_workers=2,
            ...     batch_size=1,
            ...     drop_last=True)
            ...
            >>> for data in dataloader:
            ...     print(data) # doctest: +SKIP("The output depends on the environment.")
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[2]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[6]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[3]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[7]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[4]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[8]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[5]])

    )_worker_infor   r   r   r   get_worker_infoT   s   Lr1   c                      sJ   e Zd ZU ded< ded< ded< ded< dZdd	 Z fd
dZ  ZS )r/   intnum_workersidzDataset[Any]r    seedFc                 K  s(   |  D ]
\}}t| || qd| _d S r'   )itemssetattr_WorkerInfo__initialized)r   kwargskvr   r   r   r      s   
zWorkerInfo.__init__c                   s(   | j rtd| jj dt ||S )NzCannot assign attributes to z objects)r8   RuntimeError	__class__r   super__setattr__)r   keyvalr=   r   r   r?      s
   zWorkerInfo.__setattr__)r   r   r   __annotations__r8   r   r?   __classcell__r   r   rB   r   r/      s   
 c                   @  s   e Zd ZdddZdd ZdS )_WorkerExceptionNc                 C  s2   || _ |pt }|d | _dtj| | _d S )Nr    )r   sysexc_infoexc_typejoin	tracebackformat_exceptionexc_msg)r   r   rH   r   r   r   r      s   
z_WorkerException.__init__c                 C  sB   d| j  d| jj d| j }t| jdd r| j|d| |)NzDataLoader worker(z	) caught z with message:
message)rN   )r   rI   r   rM   getattr)r   msgr   r   r   reraise   s   
z_WorkerException.reraiser   )r   r   r   r   rQ   r   r   r   r   rE      s    
rE   iװCl   u=& l   y iXl   y isI   l    c                   s   t t}fdd dd }|| t@ | d? dg} fdd|D }tt|D ]}tt|D ]}||krB|||  || ||< q/q'g }|D ]!}	|	|A t@ }
|t t@ }|
| t@ }
|
|
t? A t@ }
||
 qH|S )	Nc                   s8   |  A t @ }  t t @  |   t @ } | | t? A t @ } | S r   )MASK32MULT_AXSHIFT)value)hash_const_Ar   r   hash   s
   z_generate_states.<locals>.hashc                 S  s8   t |  t@ }t| t@ }|| t@ }||t? A t@ }|S r   )
MIX_MULT_LrS   
MIX_MULT_RrU   )xyZresult_xZresult_yresultr   r   r   mix   s
   z_generate_states.<locals>.mix    r   c                   s   g | ]} |qS r   r   ).0Zentropy)rX   r   r   
<listcomp>  s    z$_generate_states.<locals>.<listcomp>)INIT_AINIT_BrS   rangelenMULT_BrU   append)	base_seedr   Zhash_const_Br^   Z	entropiespoolijZstatespstater   )rX   rW   r   _generate_states   s(   rn   c              
     s  z]zFt t t  t| z
dd l}dd l W n	 ty%   Y nw ||	 }|	| t
	|  j	t||	 t|	|
| |dad }z|d urP||	 t|| |||}W n   t|	}Y d}t }| rHz|t}W n
 tjy~   Y qhw t|tr||d d f d}t|| ||d}qh|d u r| s|sJ dn| s|rqh|\}}z*|d ur|}d }nt
jjjt
 d ||}W d    n1 sw   Y  W n1 t y } z$t|t!r|tj"kr|t#|	 d}n
||t|	d f W Y d }~n;d }~ww t|tr|||d f t$|\}}|r<dd  fd	d
|D }||||f n||||f | smW n t%yS   Y n    W |r^t  n|rft  w w | rv|&  |'  d S d S )Nr   )r4   r3   r    r5   FTz#get None when worker done_event set)Zplacec                 S  s   t  }|| t   |S r   )r   ZDenseTensorsetCPUPlace)ZarrZ	lodtensorr   r   r   numpy2lodtensor  s   z%_worker_loop.<locals>.numpy2lodtensorc                   s(   g | ]}t | jr|n| qS r   )
isinstanceZndarrayZ
get_tensor)r`   bnprq   r   r   ra     s    

z _worker_loop.<locals>.<listcomp>)(r	   registerr
   r   Z_set_process_signal_handlerZ(_set_max_memory_map_allocation_pool_sizerandomnumpyImportErrorr5   paddlern   r/   r0   r   r$   rE   r&   r-   getr   queueEmptyrr   r   putis_setbaseZdygraphguardrp   fetch	ExceptionStopIterationr   r   r   KeyboardInterruptZcancel_join_threadclose)r    Zdataset_kindZindices_queueZ	out_queueZ
done_eventr!   r"   r#   Zinit_fnr   r3   Zuse_shared_memoryrh   Zshm_cache_sizerw   r5   Zinit_exceptionfetcherZiterator_drainedZparent_watch_dogdataidxindicesbatcheZ	structureZtensor_listr   rt   r   _worker_loop  s   










E
r   )r.   r/   )r   r   )r   ).
__future__r   r(   r|   rG   rK   typingr   r   rx   ru   rz   Z	frameworkr   Zmultiprocess_utilsr   r	   r
   r   r   r   Zflatr   Z	paddle.ior   r   r   r   r&   r0   r1   r/   rE   rb   rT   rc   rf   rY   rZ   ZdtypeZuint32itemsizerU   rS   rn   r   r   r   r   r   <module>   sB   
O.
7