
import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class   __lowerCAmelCase		(   lowerCAmelCase_	,	unittest.TestCase ):



      """simple docstring"""







      A__		:     int       				=      GPTNeoXJapaneseTokenizer
      A__		:     Dict       				=      False
      A__		:     Dict       				=      {'''do_clean_text''': False, '''add_prefix_space''': False}




      def 						snake_case_     ( self						:    Optional[Any]			):
          super().setUp()

          __lowercase						:     Optional[Any]		      =    [
              '''こん''',
              '''こんに''',
              '''にちは''',
              '''ばんは''',
              '''世界,㔺界''',
              '''、''',
              '''。''',
              '''<BR>''',
              '''<SP>''',
              '''<TAB>''',
              '''<URL>''',
              '''<EMAIL>''',
              '''<TEL>''',
              '''<DATE>''',
              '''<PRICE>''',
              '''<BLOCK>''',
              '''<KIGOU>''',
              '''<U2000U2BFF>''',
              '''<|emoji1|>''',
              '''<unk>''',
              '''<|startoftext|>''',
              '''<|endoftext|>''',
          ]
          __lowercase						:     Optional[Any]		      =    {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
          __lowercase						:     Optional[Any]		      =    {'''unk_token''': '''<unk>'''}

          __lowercase						:     Dict		      =    os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['''vocab_file''']			)
          __lowercase						:     List[Any]		      =    os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['''emoji_file''']			)
          with open(self.vocab_file			,    '''w'''			,    encoding='''utf-8'''			) as vocab_writer:
              vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)
          with open(self.emoji_file			,    '''w'''			) as emoji_writer:
              emoji_writer.write(json.dumps(_snake_case			)			)




      def 						snake_case_     ( self						:    str			,    **_snake_case						:    Tuple			):
          kwargs.update(self.special_tokens_map			)
          return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,    **_snake_case			)




      def 						snake_case_     ( self						:    Union[str, Any]			,    _snake_case						:    List[str]			):
          __lowercase						:     Tuple		      =    '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
          __lowercase						:     List[str]		      =    '''こんにちは、世界。 \nこんばんは、世界。😀'''
          return input_text, output_text




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    Tuple			):
          __lowercase						:     Any		      =    self.get_input_output_texts(_snake_case			)
          __lowercase						:     List[str]		      =    tokenizer.encode(_snake_case			,    add_special_tokens=_snake_case			)
          __lowercase						:     Dict		      =    tokenizer.decode(_snake_case			,    clean_up_tokenization_spaces=_snake_case			)
          return text, ids




      def 						snake_case_     ( self						:    Tuple			):
          pass  # TODO add if relevant




      def 						snake_case_     ( self						:    List[Any]			):
          pass  # TODO add if relevant




      def 						snake_case_     ( self						:    Tuple			):
          pass  # TODO add if relevant




      def 						snake_case_     ( self						:    Optional[int]			):
          __lowercase						:     str		      =    self.get_tokenizer()

          # Testing tokenization
          __lowercase						:     List[Any]		      =    '''こんにちは、世界。　こんばんは、㔺界。'''
          __lowercase						:     Dict		      =    ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
          __lowercase						:     Optional[int]		      =    tokenizer.tokenize(_snake_case			)
          self.assertListEqual(_snake_case			,    _snake_case			)

          # Testing conversion to ids without special tokens
          __lowercase						:     Tuple		      =    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
          __lowercase						:     Any		      =    tokenizer.convert_tokens_to_ids(_snake_case			)
          self.assertListEqual(_snake_case			,    _snake_case			)

          # Testing conversion to ids with special tokens
          __lowercase						:     Dict		      =    tokens + [tokenizer.unk_token]
          __lowercase						:     Optional[int]		      =    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
          __lowercase						:     List[str]		      =    tokenizer.convert_tokens_to_ids(_snake_case			)
          self.assertListEqual(_snake_case			,    _snake_case			)




      @slow
      def 						snake_case_     ( self						:    Union[str, Any]			):
          __lowercase						:     str		      =    self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''			)

          __lowercase						:     List[Any]		      =    tokenizer.encode('''ありがとう。'''			,    add_special_tokens=_snake_case			)
          __lowercase						:     str		      =    tokenizer.encode('''どういたしまして。'''			,    add_special_tokens=_snake_case			)

          __lowercase						:     List[str]		      =    tokenizer.build_inputs_with_special_tokens(_snake_case			)
          __lowercase						:     List[Any]		      =    tokenizer.build_inputs_with_special_tokens(_snake_case			,    _snake_case			)

          assert encoded_sentence == ids_a
          assert encoded_pair == ids_a + ids_a




      def 						snake_case_     ( self						:    List[Any]			):
          # Intentionally convert some words to accommodate character fluctuations unique to Japanese
          pass




      def 						snake_case_     ( self						:    List[Any]			):
          # tokenizer has no padding token
          pass




from ..utils import DummyObject, requires_backends


class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[str]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    int			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[str]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    int			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Tuple       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Any			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    Any			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    str			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    str			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Any			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    str			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)


