o + iH@sUddlmZddlmZmZmZmZddlmZer;ddl m Z ddl m Z ddl mZddlmZed Zd ed <dd lZdd lmZdd lZddlmZddlmZgZd%ddZd&ddZGdddeedefZ d Z!ddZ"dd Z#d!d"Z$Gd#d$d$ee%dZ&d S)') annotations) TYPE_CHECKINGAnyCallableLiteral) TypeAlias)Sequence) _DTypeLiteral) _Transform)_ImageDataType) z.jpgz.jpegz.pngz.ppmz.bmpz.pgmz.tifz.tiffz.webpr_AllowedExtensionsN)Image)Dataset) try_importfilenamestr extensions Sequence[str]returnboolcCs6t|ttfs Jdtdd|D}||S)zChecks if a file is a valid extension. Args: filename (str): path to a file extensions (list[str]|tuple[str]): extensions to consider Returns: bool: True if the filename ends with one of given extensions z#`extensions` must be list or tuple.cSsg|]}|qS)lower).0xrri/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/paddle/vision/datasets/folder.py Az'has_valid_extension..) isinstancelisttuplerendswith)rrrrrhas_valid_extension4s  r"c sg}tj|}durfdd}t|D]<}tj||}tj|s(qttj|ddD]"\}}} t| D]} tj|| } || rR| ||f} || q:q1q|S)Nc t|SNr"rrrr is_valid_fileK z#make_dataset..is_valid_fileT followlinks) ospath expandusersortedkeysjoinisdirwalkappend) dir class_to_idxrr(Zimagestargetdroot_fnamesfnamer-itemrr'r make_datasetEs$      r>c@seZdZUdZded<ded<ded<ded <d ed <d ed <ded<ded<    d%d&ddZd'ddZd(d!d"Zd#d$ZdS)) DatasetFoldera"A generic data loader where the samples are arranged in this way: .. code-block:: text root/class_a/1.ext root/class_a/2.ext root/class_a/3.ext root/class_b/123.ext root/class_b/456.ext root/class_b/789.ext Args: root (str): Root directory path. loader (Callable|None, optional): A function to load a sample given its path. Default: None. extensions (list[str]|tuple[str]|None, optional): A list of allowed extensions. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. If this value is not set, the default is to use ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp'). Default: None. transform (Callable|None, optional): A function/transform that takes in a sample and returns a transformed version. Default: None. is_valid_file (Callable|None, optional): A function that takes path of a file and check if the file is a valid file. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. Default: None. Returns: :ref:`api_paddle_io_Dataset`. An instance of DatasetFolder. Attributes: classes (list[str]): List of the class names. class_to_idx (dict[str, int]): Dict with items (class_name, class_index). samples (list[tuple[str, int]]): List of (sample_path, class_index) tuples. targets (list[int]): The class_index value for each image in the dataset. Example: .. code-block:: python >>> import shutil >>> import tempfile >>> import cv2 >>> import numpy as np >>> import paddle.vision.transforms as T >>> from pathlib import Path >>> from paddle.vision.datasets import DatasetFolder >>> def make_fake_file(img_path: str): ... if img_path.endswith((".jpg", ".png", ".jpeg")): ... fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) ... cv2.imwrite(img_path, fake_img) ... elif img_path.endswith(".txt"): ... with open(img_path, "w") as f: ... f.write("This is a fake file.") >>> def make_directory(root, directory_hierarchy, file_maker=make_fake_file): ... root = Path(root) ... root.mkdir(parents=True, exist_ok=True) ... for subpath in directory_hierarchy: ... if isinstance(subpath, str): ... filepath = root / subpath ... file_maker(str(filepath)) ... else: ... dirname = list(subpath.keys())[0] ... make_directory(root / dirname, subpath[dirname]) >>> directory_hierarchy = [ ... {"class_0": [ ... "abc.jpg", ... "def.png"]}, ... {"class_1": [ ... "ghi.jpeg", ... "jkl.png", ... {"mno": [ ... "pqr.jpeg", ... "stu.jpg"]}]}, ... "this_will_be_ignored.txt", ... ] >>> # You can replace this with any directory to explore the structure >>> # of generated data. e.g. fake_data_dir = "./temp_dir" >>> fake_data_dir = tempfile.mkdtemp() >>> make_directory(fake_data_dir, directory_hierarchy) >>> data_folder_1 = DatasetFolder(fake_data_dir) >>> print(data_folder_1.classes) ['class_0', 'class_1'] >>> print(data_folder_1.class_to_idx) {'class_0': 0, 'class_1': 1} >>> print(data_folder_1.samples) >>> # doctest: +SKIP(it's different with windows) [('./temp_dir/class_0/abc.jpg', 0), ('./temp_dir/class_0/def.png', 0), ('./temp_dir/class_1/ghi.jpeg', 1), ('./temp_dir/class_1/jkl.png', 1), ('./temp_dir/class_1/mno/pqr.jpeg', 1), ('./temp_dir/class_1/mno/stu.jpg', 1)] >>> # doctest: -SKIP >>> print(data_folder_1.targets) [0, 0, 1, 1, 1, 1] >>> print(len(data_folder_1)) 6 >>> for i in range(len(data_folder_1)): ... img, label = data_folder_1[i] ... # do something with img and label ... print(type(img), img.size, label) ... # (32, 32) 0 >>> transform = T.Compose( ... [ ... T.Resize(64), ... T.ToTensor(), ... T.Normalize( ... mean=[0.5, 0.5, 0.5], ... std=[0.5, 0.5, 0.5], ... to_rgb=True, ... ), ... ] ... ) >>> data_folder_2 = DatasetFolder( ... fake_data_dir, ... loader=lambda x: cv2.imread(x), # load image with OpenCV ... extensions=(".jpg",), # only load *.jpg files ... transform=transform, # apply transform to every image ... ) >>> print([img_path for img_path, label in data_folder_2.samples]) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/class_0/abc.jpg', './temp_dir/class_1/mno/stu.jpg'] >>> # doctest: -SKIP >>> print(len(data_folder_2)) 2 >>> for img, label in iter(data_folder_2): ... # do something with img and label ... print(type(img), img.shape, label) # type: ignore ... # [3, 64, 64] 0 >>> shutil.rmtree(fake_data_dir) $Callable[..., _ImageDataType] | Noneloader#Sequence[_AllowedExtensions] | Noner_Transform[Any, Any] | None transform list[str]classeszdict[str, int]r6zlist[tuple[str, int]]samplestargetsr dtypeNr9rr(_ImageDataType | NonerNonec Cs||_||_|dur t}||j\}}t|j|||}t|dkr0td|jdd||dur6tn||_ ||_ ||_ ||_ ||_ dd|D|_t|_dS)Nrz&Found 0 directories in subfolders of:  Supported extensions are: ,cSsg|]}|dqS)r)rsrrrrrz*DatasetFolder.__init__..)r9rDIMG_EXTENSIONS _find_classesr>len RuntimeErrorr1default_loaderrArrFr6rGrHpaddleZget_default_dtyperI) selfr9rArrDr(rFr6rGrrr__init__s,   zDatasetFolder.__init__r5 tuple[list[str], dict[str, int]]cs>ddt|DfddttD}|fS)a Finds the class folders in a dataset. Args: dir (string): Root directory path. Returns: tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. cSsg|] }|r|jqSr)is_dirname)rr8rrrrsz/DatasetFolder._find_classes..csi|]}||qSrr)rirFrr !sz/DatasetFolder._find_classes..)r,scandirsortrangerR)rVr5r6rr\rrQs zDatasetFolder._find_classesindexinttuple[_ImageDataType, int]cCs4|j|\}}||}|jdur||}||fS)z Args: index (int): Index Returns: tuple: (sample, target) where target is class_index of the target class. NrGrArD)rVrar-r7samplerrr __getitem__$s    zDatasetFolder.__getitem__cC t|jSr$rRrGrVrrr__len__3r)zDatasetFolder.__len__NNNN r9rrAr@rrBrDrCr(rJrrK)r5rrrX)rarbrrc) __name__ __module__ __qualname____doc____annotations__rWrQrfrjrrrrr?\s&   "  r?r cCsBt|d}t|}|dWdS1swYdS)NrbRGB)openrconvert)r-fimgrrr pil_loaderDs  $rxcCstd}||||jS)Ncv2)rZcvtColorZimreadZ COLOR_BGR2RGB)r-ryrrr cv2_loaderJsrzcCs&ddlm}|dkrt|St|S)Nr)get_image_backendry)Z paddle.visionr{rzrx)r-r{rrrrTOs  rTc@sZeZdZUdZded<ded<ded<ded < ddddZdddZdddZd S) ImageFolderamA generic data loader where the samples are arranged in this way: .. code-block:: text root/1.ext root/2.ext root/sub_dir/3.ext Args: root (str): Root directory path. loader (Callable|None, optional): A function to load a sample given its path. Default: None. extensions (list[str]|tuple[str]|None, optional): A list of allowed extensions. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. If this value is not set, the default is to use ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp'). Default: None. transform (Callable|None, optional): A function/transform that takes in a sample and returns a transformed version. Default: None. is_valid_file (Callable|None, optional): A function that takes path of a file and check if the file is a valid file. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. Default: None. Returns: :ref:`api_paddle_io_Dataset`. An instance of ImageFolder. Attributes: samples (list[str]): List of sample path. Example: .. code-block:: python >>> import shutil >>> import tempfile >>> import cv2 >>> import numpy as np >>> import paddle.vision.transforms as T >>> from pathlib import Path >>> from paddle.vision.datasets import ImageFolder >>> def make_fake_file(img_path: str): ... if img_path.endswith((".jpg", ".png", ".jpeg")): ... fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) ... cv2.imwrite(img_path, fake_img) ... elif img_path.endswith(".txt"): ... with open(img_path, "w") as f: ... f.write("This is a fake file.") >>> def make_directory(root, directory_hierarchy, file_maker=make_fake_file): ... root = Path(root) ... root.mkdir(parents=True, exist_ok=True) ... for subpath in directory_hierarchy: ... if isinstance(subpath, str): ... filepath = root / subpath ... file_maker(str(filepath)) ... else: ... dirname = list(subpath.keys())[0] ... make_directory(root / dirname, subpath[dirname]) >>> directory_hierarchy = [ ... "abc.jpg", ... "def.png", ... {"ghi": [ ... "jkl.jpeg", ... {"mno": [ ... "pqr.jpg"]}]}, ... "this_will_be_ignored.txt", ... ] >>> # You can replace this with any directory to explore the structure >>> # of generated data. e.g. fake_data_dir = "./temp_dir" >>> fake_data_dir = tempfile.mkdtemp() >>> make_directory(fake_data_dir, directory_hierarchy) >>> image_folder_1 = ImageFolder(fake_data_dir) >>> print(image_folder_1.samples) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/abc.jpg', './temp_dir/def.png', './temp_dir/ghi/jkl.jpeg', './temp_dir/ghi/mno/pqr.jpg'] >>> # doctest: -SKIP >>> print(len(image_folder_1)) 4 >>> for i in range(len(image_folder_1)): ... (img,) = image_folder_1[i] ... # do something with img ... print(type(img), img.size) ... # (32, 32) >>> transform = T.Compose( ... [ ... T.Resize(64), ... T.ToTensor(), ... T.Normalize( ... mean=[0.5, 0.5, 0.5], ... std=[0.5, 0.5, 0.5], ... to_rgb=True, ... ), ... ] ... ) >>> image_folder_2 = ImageFolder( ... fake_data_dir, ... loader=lambda x: cv2.imread(x), # load image with OpenCV ... extensions=(".jpg",), # only load *.jpg files ... transform=transform, # apply transform to every image ... ) >>> print(image_folder_2.samples) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/abc.jpg', './temp_dir/ghi/mno/pqr.jpg'] >>> # doctest: -SKIP >>> print(len(image_folder_2)) 2 >>> for (img,) in iter(image_folder_2): ... # do something with img ... print(type(img), img.shape) # type: ignore ... # [3, 64, 64] >>> shutil.rmtree(fake_data_dir) r@rArBrrErGrCrDNr9rr(rJrrKc s||_dur tg}tj|}durfdd}ttj|ddD]\}}} t| D]} tj|| } || r?|| q-q$t |dkrUt d|jdd|dur[t n||_ |_ ||_||_dS) Ncr#r$r%r&r'rrr(r)z+ImageFolder.__init__..is_valid_fileTr*rz Found 0 files in subfolders of: rLrM)r9rPr,r-r.r/r3r1r4rRrSrTrArrGrD) rVr9rArrDr(rGr-r:r;r<rvrr'rrWs2       zImageFolder.__init__rarblist[_ImageDataType]cCs.|j|}||}|jdur||}|gS)zn Args: index (int): Index Returns: sample of specific index. Nrd)rVrar-rerrrrfs    zImageFolder.__getitem__cCrgr$rhrirrrrj r)zImageFolder.__len__rkrl)rarbrr})rrb)rmrnrorprqrWrfrjrrrrr|Xs z 'r|)rrrrrrr$)' __future__rtypingrrrrZtyping_extensionsrcollections.abcrZpaddle._typing.dtype_liker Z#paddle.vision.transforms.transformsr imager r rqr,ZPILrrUZ paddle.iorZ paddle.utilsr__all__r"r>r rbr?rPrxrzrTrr|rrrrs4           \