\;j1 f d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm Z g Z
dZ dZ G d de Z
y) N)_check_exists_and_download)Datasetz6https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz 7c2ac02c03563afcf9b574c7e56c153ac 6 e Zd ZdZd dZd Zd Zd Zd Zd Z y)
Imdba
Implementation of `IMDB `_ dataset.
Args:
data_file(str): path to data tar file, can be set None if
:attr:`download` is True. Default None
mode(str): 'train' 'test' mode. Default 'train'.
cutoff(int): cutoff number for building word dictionary. Default 150.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True
Returns:
Dataset: instance of IMDB dataset
Examples:
.. code-block:: python
>>> # doctest: +TIMEOUT(75)
>>> import paddle
>>> from paddle.text.datasets import Imdb
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, doc, label):
... return paddle.sum(doc), label
>>> imdb = Imdb(mode='train')
>>> for i in range(10):
... doc, label = imdb[i]
... doc = paddle.to_tensor(doc)
... label = paddle.to_tensor(label)
...
... model = SimpleNet()
... image, label = model(doc, label)
... print(doc.shape, label.shape)
[121] [1]
[115] [1]
[386] [1]
[471] [1]
[585] [1]
[206] [1]
[221] [1]
[324] [1]
[166] [1]
[598] [1]
Nc " |j dv s
J d| |j | _ || _ | j %|sJ d t |t t
d| | _ | j
| | _ | j y )N)traintestz(mode should be 'train', 'test', but got z>data_file is not set and downloading automatically is disabledimdb) lowermode data_filer URLMD5_build_work_dictword_idx
_load_anno)selfr r
cutoffdownloads bG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/text/datasets/imdb.py__init__z
Imdb.__init__U s zz|
= 6dV < =
JJL " >> !
PO
P 73VX DN
--f5
c t j t }t j d }| j | D ] }|D ] }||xx dz
cc< |j
D cg c]
}|d |kD s | }}t |d }t t | \ }} t t t |t t | }
t | |
d< |
S c c}w )Nz/aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$ c | d | d fS )Nr r )xs r z'Imdb._build_work_dict..u s qteQqT]r )key)
collections defaultdictintrecompile _tokenizeitemssortedlistzipdictrangelen) r r word_freqpatterndocwordr
dictionarywords_r s r r zImdb._build_work_dictk s ++C0 ** OP>>'*C$1$ +
!* 1C 11QqTF]Q 1 CI+BC
Z()qSc%j(9 :;< J
Ds +
C!9C!c g }t j | j 5 }|j }|t |j |j r|j |j | j j d j d t j j d j j! |j }|d d d |S # 1 sw Y |S xY w)Ns
zlatin-1)tarfileopenr nextboolmatchnameappend extractfilereadrstrip translatestring punctuationencoder split)r r0 datatarftfs r r' zImdb._tokenize{ s
\\$..
)TB.
bgg./KK ((, "4););)B)B9)MN
YY[ . * * s CC<<Dc
t j d| j d }t j d| j d }| j d }g | _ g | _ | j
| D ]` }| j j |D cg c] }| j j || c} | j
j d b | j
| D ]` }| j j |D cg c] }| j j || c} | j
j d b y c c}w c c}w )NzaclImdb/z
/pos/.*\.txt$z
/neg/.*\.txt$r! r r ) r% r& r
r docslabelsr' r= get)r pos_pattern neg_patternUNKr1 ws r r zImdb._load_anno s jjHTYYK}!EF jjHTYYK}!EF mmG$ >>+.C II EAdmm / /3 7 E F KK q ! / >>+.C II EAdmm / /3 7 E F KK q ! / F Fs #E
#E
c t j | j | t j | j | g fS N)nparrayrJ rK )r idxs r __getitem__zImdb.__getitem__ s3 3 ("((DKK4D3E*FGGr c , t | j S rR )r. rJ )r s r __len__z Imdb.__len__ s 499~ r )Nr T)
__name__
__module__ __qualname____doc__r r r' r rV rX r r r r r s' 3j, &
" H r r )r" r% rB r7 numpyrS paddle.dataset.commonr paddle.ior __all__r r r r r r rb s9
<
>(B 7 B r