Ë
    –\;j)  ã                   ó‚   — d Z ddlZddlZddlmZ ddlZddlZddlm	Z	 ddl
mZ g ZdZdZdZd	Zd
ZdZdZ G d„ de«      Zy)Ú
é    N)Údefaultdict)Ú_check_exists_and_download)ÚDatasetz2http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gzÚ 0c38be43600334966403524a40dcd81eiò+  iK  z<s>z<e>z<unk>c                   óL   — e Zd ZdZ	 	 	 	 	 	 d
d„Zdd„Zd„ Zd„ Zd„ Zd„ Z	dd	„Z
y)ÚWMT16a{  
    Implementation of `WMT16 <http://www.statmt.org/wmt16/>`_ test dataset.
    ACL2016 Multimodal Machine Translation. Please see this website for more
    details: http://www.statmt.org/wmt16/multimodal-task.html#task1

    If you use the dataset created for your task, please cite the following paper:
    Multi30K: Multilingual English-German Image Descriptions.

    .. code-block:: text

        @article{elliott-EtAl:2016:VL16,
         author    = {{Elliott}, D. and {Frank}, S. and {Sima"an}, K. and {Specia}, L.},
         title     = {Multi30K: Multilingual English-German Image Descriptions},
         booktitle = {Proceedings of the 6th Workshop on Vision and Language},
         year      = {2016},
         pages     = {70--74},
         year      = 2016
        }

    Args:
        data_file(str): path to data tar file, can be set None if
            :attr:`download` is True. Default None.
        mode(str): 'train', 'test' or 'val'. Default 'train'.
        src_dict_size(int): word dictionary size for source language word. Default -1.
        trg_dict_size(int): word dictionary size for target language word. Default -1.
        lang(str): source language, 'en' or 'de'. Default 'en'.
        download(bool): whether to download dataset automatically if
            :attr:`data_file` is not set. Default True.

    Returns:
        Dataset: Instance of WMT16 dataset. The instance of dataset has 3 fields:
            - src_ids (np.array) - The sequence of token ids of source language.
            - trg_ids (np.array) - The sequence of token ids of target language.
            - trg_ids_next (np.array) - The next sequence of token ids of target language.

    Examples:

        .. code-block:: python

            >>> import paddle
            >>> from paddle.text.datasets import WMT16

            >>> class SimpleNet(paddle.nn.Layer):
            ...     def __init__(self):
            ...         super().__init__()
            ...
            ...     def forward(self, src_ids, trg_ids, trg_ids_next):
            ...         return paddle.sum(src_ids), paddle.sum(trg_ids), paddle.sum(trg_ids_next)

            >>> wmt16 = WMT16(mode='train', src_dict_size=50, trg_dict_size=50)

            >>> for i in range(10):
            ...     src_ids, trg_ids, trg_ids_next = wmt16[i]
            ...     src_ids = paddle.to_tensor(src_ids)
            ...     trg_ids = paddle.to_tensor(trg_ids)
            ...     trg_ids_next = paddle.to_tensor(trg_ids_next)
            ...
            ...     model = SimpleNet()
            ...     src_ids, trg_ids, trg_ids_next = model(src_ids, trg_ids, trg_ids_next)
            ...     print(src_ids.item(), trg_ids.item(), trg_ids_next.item())
            89 32 33
            79 18 19
            55 26 27
            147 36 37
            106 22 23
            135 50 51
            54 43 44
            217 30 31
            146 51 52
            55 24 25
    Nc                 ó(  — |j                  «       dv s
J d|› �«       ‚|j                  «       | _        || _        | j                  €%|sJ d«       ‚t        |t        t
        d|«      | _        || _        |dkD  sJ d«       ‚|dkD  sJ d«       ‚t        ||dk(  rt        nt        «      | _
        t        ||dk(  rt        nt        «      | _        | j                  ||«      | _        | j                  |dk(  rdnd|«      | _        | j                  «       | _        y )	N)ÚtrainÚtestÚvalz1mode should be 'train', 'test' or 'val', but got z>data_file is not set and downloading automatically is disabledÚwmt16r   z*dict_size should be set as positive numberÚenÚde)ÚlowerÚmodeÚ	data_filer   ÚDATA_URLÚDATA_MD5ÚlangÚminÚTOTAL_EN_WORDSÚTOTAL_DE_WORDSÚsrc_dict_sizeÚtrg_dict_sizeÚ
_load_dictÚsrc_dictÚtrg_dictÚ
_load_dataÚdata)Úselfr   r   r   r   r   Údownloads          úcG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/text/datasets/wmt16.pyÚ__init__zWMT16.__init__q   s(  € ð �z‰z‹|ð  
ñ 
ð 	Fð ?¸t¸fÐEó		Fð 
ð
 —J‘J“LˆŒ	à"ˆŒØ�>‰>Ð!áðPàOóPØä7Øœ8¤X¨w¸óˆDŒNð ˆŒ	Ø˜qÒ ÐNÐ"NÓNÐ Ø˜qÒ ÐNÐ"NÓNÐ Ü Ø¨d°dªl�NÄó
ˆÔô !Ø¨d°dªl�NÄó
ˆÔð
 Ÿ™¨¨mÓ<ˆŒØŸ™Ø˜D’L‰D d¨Mó
ˆŒð
 —O‘OÓ%ˆ�	ó    c                 ót  — t         j                  j                  t        j                  j
                  j                  d||fz  «      }d}t         j                  j                  |«      r1t        |d«      5 }t        |j                  «       «      |k(  }d d d «       |s| j                  |||«       i }t        |d«      5 }t        |«      D ]J  \  }	}
|r"|
j                  «       j                  «       ||	<   Œ*|	||
j                  «       j                  «       <   ŒL 	 d d d «       |S # 1 sw Y   Œ�xY w# 1 sw Y   |S xY w)Núwmt16/%s_%d.dictFÚrb)ÚosÚpathÚjoinÚpaddleÚdatasetÚcommonÚ	DATA_HOMEÚexistsÚopenÚlenÚ	readlinesÚ_build_dictÚ	enumerateÚstripÚdecode)r!   r   Ú	dict_sizeÚreverseÚ	dict_pathÚ
dict_foundÚdÚ	word_dictÚfdictÚidxÚlines              r#   r   zWMT16._load_dict�   s  € Ü—G‘G—L‘LÜ�N‰N×!Ñ!×+Ñ+Ø $¨	Ð!2Ñ2ó
ˆ	ð ˆ
Ü�7‰7�>‰>˜)Ô$Ü�i Ô&¨!Ü  §¡£Ó/°9Ñ<�
÷ 'áØ×Ñ˜Y¨	°4Ô8àˆ	Ü�)˜TÔ" eÜ& uÖ-‘	��TÙØ%)§Z¡Z£\×%8Ñ%8Ó%:�I˜c’Nà7:�I˜dŸj™j›l×1Ñ1Ó3Ò4ñ	 .÷ #ð Ð÷ 'Ð&ú÷ #ð Ðús   Á5D!Â=AD-Ä!D*Ä-D7c           	      ó.  — t        t        «      }t        j                  | j                  d¬«      5 }|j                  d«      D ]{  }|j                  «       }|j                  «       j                  d«      }t        |«      dk7  rŒA| j                  dk(  r|d   n|d   }|j                  «       D ]  }	||	xx   dz  cc<   Œ Œ} 	 d d d «       t        |d	«      5 }
|
j                  t        › d
t        › d
t        › d
�j                  «       «       t!        t#        |j%                  «       d„ d¬«      «      D ]B  \  }}|dz   |k(  r n5|
j                  |d   j                  «       «       |
j                  d«       ŒD d d d «       y # 1 sw Y   Œ¾xY w# 1 sw Y   y xY w)NÚr©r   zwmt16/trainÚ	é   r   r   é   Úwbr   c                 ó   — | d   S )NrF   © )Úxs    r#   Ú<lambda>z#WMT16._build_dict.<locals>.<lambda>Á   s   € ¸¸!ºr%   T)Úkeyr9   é   ó   
)r   ÚintÚtarfiler1   r   Úextractfiler7   r6   Úsplitr2   r   ÚwriteÚ
START_MARKÚEND_MARKÚUNK_MARKÚencoder5   ÚsortedÚitems)r!   r:   r8   r   r=   Úfr@   Ú
line_splitÚsenÚwÚfoutr?   Úwords                r#   r4   zWMT16._build_dict²   sJ  € Ü¤Ó$ˆ	Ü�\‰\˜$Ÿ.™.¨sÕ3°qØŸ™ mÖ4�Ø—{‘{“}�Ø!ŸZ™Z›\×/Ñ/°Ó5�
Ü�z“? aÒ'ØØ'+§y¡y°DÒ'8�j ’m¸jÈ¹m�ØŸ™ž�AØ˜a“L AÑ%”Lñ %ñ 5÷ 4ô �)˜TÔ" dØ�J‰Jœ:˜, b¬¨
°"´X°J¸bÐA×IÑIÓKÔLÜ&Ü�y—‘Ó(©nÀdÔKö‘	��Tð ˜‘7˜iÒ'ÙØ—
‘
˜4 ™7Ÿ>™>Ó+Ô,Ø—
‘
˜5Õ!ð÷ #Ð"÷ 4Ð3ú÷ #Ð"ús   ±BE?ÃB FÅ?FÆFc           
      óÔ  — | j                   t           }| j                   t           }| j                   t           }| j                  dk(  rdnd}d|z
  }g | _        g | _        g | _        t        j                  | j                  d¬«      5 }|j                  d| j                  › �«      D �]  }|j                  «       }|j                  «       j                  d«      }t!        |«      dk7  rŒB||   j                  «       }	|g|	D �
cg c]  }
| j                   j#                  |
|«      ‘Œ  c}
z   |gz   }||   j                  «       }|D �
cg c]  }
| j$                  j#                  |
|«      ‘Œ  }}
||gz   }|g|z   }| j
                  j'                  |«       | j                  j'                  |«       | j                  j'                  |«       �Œ" 	 d d d «       y c c}
w c c}
w # 1 sw Y   y xY w)	Nr   r   rF   rB   rC   zwmt16/rD   rE   )r   rT   rU   rV   r   Úsrc_idsÚtrg_idsÚtrg_ids_nextrP   r1   r   rQ   r   r7   r6   rR   r2   Úgetr   Úappend)r!   Ústart_idÚend_idÚunk_idÚsrc_colÚtrg_colrZ   r@   r[   Ú	src_wordsr]   ra   Ú	trg_wordsrb   rc   s                  r#   r   zWMT16._load_dataÈ   s¸  € ð —=‘=¤Ñ,ˆØ—‘œxÑ(ˆØ—‘œxÑ(ˆà—y‘y DÒ(‘!¨aˆØ�g‘+ˆàˆŒØˆŒØˆÔÜ�\‰\˜$Ÿ.™.¨sÕ3°qØŸ™¨¨t¯y©y¨kÐ&:×;�Ø—{‘{“}�Ø!ŸZ™Z›\×/Ñ/°Ó5�
Ü�z“? aÒ'ØØ& wÑ/×5Ñ5Ó7�	à�JÙ=FÓG¹Y¸�t—}‘}×(Ñ(¨¨FÕ3¸YÑGñHà�hñð ð ' wÑ/×5Ñ5Ó7�	ÙAJÓKÁ¸A˜4Ÿ=™=×,Ñ,¨Q°Õ7À�ÐKà&¨&¨Ñ1�Ø#˜* wÑ.�à—‘×#Ñ# GÔ,Ø—‘×#Ñ# GÔ,Ø×!Ñ!×(Ñ(¨Ö6ñ) <÷ 4Ð3ùò Hùò
 L÷ 4Ð3ús,   ÂA:GÄ#G
Ä% GÅ#GÅ(A"GÇ
GÇG'c                 óÌ   — t        j                  | j                  |   «      t        j                  | j                  |   «      t        j                  | j                  |   «      fS ©N)ÚnpÚarrayra   rb   rc   )r!   r?   s     r#   Ú__getitem__zWMT16.__getitem__í   sM   € ä�H‰H�T—\‘\ #Ñ&Ó'Ü�H‰H�T—\‘\ #Ñ&Ó'Ü�H‰H�T×&Ñ& sÑ+Ó,ð
ð 	
r%   c                 ó,   — t        | j                  «      S rn   )r2   ra   )r!   s    r#   Ú__len__zWMT16.__len__ô   s   € Ü�4—<‘<Ó Ð r%   c                 óR  — || j                   k(  r| j                  n| j                  }t        j                  j                  t        j                  j                  j                  d||fz  «      }t        j                  j                  |«      sJ d«       ‚	 	 | j                  ||«      S )a£  
        return the word dictionary for the specified language.

        Args:
            lang(string): A string indicating which language is the source
                          language. Available options are: "en" for English
                          and "de" for Germany.
            reverse(bool): If reverse is set to False, the returned python
                           dictionary will use word as key and use index as value.
                           If reverse is set to True, the returned python
                           dictionary will use index as key and word as value.

        Returns:
            dict: The word dictionary for the specific language.

        Examples:

            .. code-block:: python

                >>> from paddle.text.datasets import WMT16
                >>> wmt16 = WMT16(mode='train', src_dict_size=50, trg_dict_size=50)
                >>> en_dict = wmt16.get_dict('en')

        r'   z Word dictionary does not exist. )r   r   r   r)   r*   r+   r,   r-   r.   r/   r0   r   )r!   r   r9   r8   r:   s        r#   Úget_dictzWMT16.get_dict÷   sŽ   € ð4 #'¨$¯)©)Ò"3ˆD×Ò¸×9KÑ9Kð 	ô —G‘G—L‘LÜ�N‰N×!Ñ!×+Ñ+Ø $¨	Ð!2Ñ2ó
ˆ	ô �w‰w�~‰~˜iÔ(ÐLÐ*LÓLÐ(ØIØ"Ø�‰˜t YÓ/Ð/r%   )Nr   éÿÿÿÿrv   r   T)F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   r   r4   r   rq   rs   ru   rI   r%   r#   r	   r	   (   sB   „ ñFðT ØØØØØó*&óXò*"ò,#7òJ
ò!ô$0r%   r	   )rz   r)   rP   Úcollectionsr   Únumpyro   r,   Úpaddle.dataset.commonr   Ú	paddle.ior   Ú__all__r   r   r   r   rT   rU   rV   r	   rI   r%   r#   Ú<module>r€      sW   ðñó 
Û Ý #ã ã Ý <Ý à
€à?€Ø-€à€Ø€à€
Ø€Ø€ôs0ˆGõ s0r%   