o
p i @ s d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl Z
d dl m Z d dl
mZ er@d dlmZ d dlmZ ed Zg ZdZd ZG d
d d eZdS ) ) annotationsN)
TYPE_CHECKINGLiteral)_check_exists_and_download)Dataset)Patterntraintestz6https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gzZ 7c2ac02c03563afcf9b574c7e56c153ac @ s e Zd ZU dZded< ded< ded< ded < ded
<
d&d'ddZd(ddZd)d d Zd*d d Zd+d"d#Z d,d$d%Z
d S )-Imdba
Implementation of `IMDB `_ dataset.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train' 'test' mode. Default 'train'.
cutoff(int): cutoff number for building word dictionary. Default 150.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: instance of IMDB dataset
Examples:
.. code-block:: python
>>> # doctest: +TIMEOUT(75)
>>> import paddle
>>> from paddle.text.datasets import Imdb
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, doc, label):
... return paddle.sum(doc), label
>>> imdb = Imdb(mode='train')
>>> for i in range(10):
... doc, label = imdb[i]
... doc = paddle.to_tensor(doc)
... label = paddle.to_tensor(label)
...
... model = SimpleNet()
... image, label = model(doc, label)
... print(doc.shape, label.shape)
[121] [1]
[115] [1]
[386] [1]
[471] [1]
[585] [1]
[206] [1]
[221] [1]
[324] [1]
[166] [1]
[598] [1]
str | None data_file_ImdbDataSetModemodedict[str, int]word_idxlistdocslabelsNr TcutoffintdownloadboolreturnNonec C sj | dv s
J d| | | _|| _| jd u r)|s J dt|ttd|| _| || _| d S )Nr z(mode should be 'train', 'test', but got z>data_file is not set and downloading automatically is disabledZimdb) lowerr r
r URLMD5_build_work_dictr
_load_anno)selfr
r r r r" `/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddle/text/datasets/imdb.py__init__c s
z
Imdb.__init__c
s t t}td}| |D ]}|D ]
}|| d7 < qq fdd| D }t|dd d}tt | \}}t
tt |t t |} t || d< | S ) Nz/aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$ c s g | ]
}|d kr|qS )r% r" ).0xr r" r#
s z)Imdb._build_work_dict..c S s | d | d fS )Nr% r r" )r' r" r" r# s z'Imdb._build_work_dict..)key)
collections defaultdictr recompile _tokenizeitemssortedr zipdictrangelen)
r! r Z word_freqpatterndocword
dictionarywords_r r" r( r# r s
zImdb._build_work_dictr8 Pattern[str]list[list[str]]c C s g }t | jB}| }|d urBt||jr2|||
d d t j
d | }|d usW d |S W d |S 1 sMw Y |S )Ns
zlatin-1)tarfileopenr
nextr matchnameappend extractfilereadrstrip translatestring punctuationencoder split)r! r8 dataZtarftfr" r" r# r1 s,
zImdb._tokenizec s t dj d}t dj d}jd g _g _|D ]}j fdd|D jd q$|D ]}j fdd|D jd q@d S )
NzaclImdb/z
/pos/.*\.txt$z
/neg/.*\.txt$r, c g | ] }j | qS r" r getr& wZUNKr! r" r# r) z#Imdb._load_anno..r c rP r" rQ rS rU r" r# r) rV r% )r/ r0 r r r r r1 rE )r! Z pos_patternZ neg_patternr9 r" rU r# r s
zImdb._load_annoidx1tuple[npt.NDArray[np.int_], npt.NDArray[np.int_]]c C s" t | j| t | j| gfS N)nparrayr r )r! rW r" r" r# __getitem__ s "zImdb.__getitem__c C s
t | jS rY )r7 r )r! r" r" r# __len__ s
z Imdb.__len__)Nr r T)
r
r r r r r r r r r )r r r r )r8 r> r r? )r r )rW r r rX )r r ) __name__
__module__ __qualname____doc____annotations__r$ r r1 r r\ r] r" r" r" r# r ' s"
5
r )
__future__r r- r/ rJ r@ typingr r numpyrZ Zpaddle.dataset.commonr Z paddle.ior r Z numpy.typingZnptr __all__r r r r" r" r" r# s"