
import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class   __lowerCAmelCase		(   lowercase__	,	unittest.TestCase ):



      """simple docstring"""







      A__		:     Dict       				=      BertweetTokenizer
      A__		:     Union[str, Any]       				=      False




      def 						snake_case_     ( self						:    Union[str, Any]			):
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          __lowercase						:     Optional[Any]		      =    ['''I''', '''m''', '''V@@''', '''R@@''', '''r''', '''e@@''']
          __lowercase						:     int		      =    dict(zip(lowerCAmelCase__			,    range(len(lowerCAmelCase__			)			)			)			)
          __lowercase						:     Optional[int]		      =    ['''#version: 0.2''', '''a m</w>''']
          __lowercase						:     str		      =    {'''unk_token''': '''<unk>'''}

          __lowercase						:     int		      =    os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['''vocab_file''']			)
          __lowercase						:     str		      =    os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['''merges_file''']			)
          with open(self.vocab_file			,    '''w'''			,    encoding='''utf-8'''			) as fp:
              for token in vocab_tokens:
                  fp.write(F'{token} {vocab_tokens[token]}\n'			)
          with open(self.merges_file			,    '''w'''			,    encoding='''utf-8'''			) as fp:
              fp.write('''\n'''.join(lowerCAmelCase__			)			)




      def 						snake_case_     ( self						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          kwargs.update(self.special_tokens_map			)
          return BertweetTokenizer.from_pretrained(self.tmpdirname			,    **lowerCAmelCase__			)




      def 						snake_case_     ( self						:    Tuple			,    _snake_case						:    Optional[Any]			):
          __lowercase						:     int		      =    '''I am VinAI Research'''
          __lowercase						:     Union[str, Any]		      =    '''I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
          return input_text, output_text




      def 						snake_case_     ( self						:    int			):
          __lowercase						:     Any		      =    BertweetTokenizer(self.vocab_file			,    self.merges_file			,    **self.special_tokens_map			)
          __lowercase						:     Tuple		      =    '''I am VinAI Research'''
          __lowercase						:     List[str]		      =    '''I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
          __lowercase						:     str		      =    tokenizer.tokenize(lowerCAmelCase__			)
          self.assertListEqual(lowerCAmelCase__			,    lowerCAmelCase__			)

          __lowercase						:     Optional[Any]		      =    tokens + [tokenizer.unk_token]

          __lowercase						:     Union[str, Any]		      =    [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)			,    lowerCAmelCase__			)




from ..utils import DummyObject, requires_backends


class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[str]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    int			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[str]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    int			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Tuple       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Any			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    Any			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    str			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    str			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Any			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    str			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)


