
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase   :     str      =						logging.get_logger(__name__)

__lowerCAmelCase   :     Union[str, Any]      =						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__lowerCAmelCase   :     Dict      =						{
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
}

__lowerCAmelCase   :     List[str]      =						{
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}


@lru_cache()
def       UpperCAmelCase_					(  )	->  Union[str, Any]:
    __lowercase						:     Union[str, Any]		      =    (
        list(range(ord('''!'''		)		,   ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)		,   ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)		,   ord('''ÿ'''		) + 1		)		)
    )
    __lowercase						:     int		      =    bs[:]
    __lowercase						:     Any		      =    0
    for b in range(2**8		):
        if b not in bs:
            bs.append(__lowerCAmelCase		)
            cs.append(2**8 + n		)
            n += 1
    __lowercase						:     Optional[Any]		      =    [chr(__lowerCAmelCase		) for n in cs]
    return dict(zip(__lowerCAmelCase		,   __lowerCAmelCase		)		)


def       UpperCAmelCase_					(  __lowerCAmelCase		)	->  Optional[Any]:
    __lowercase						:     List[Any]		      =    set()
    __lowercase						:     Union[str, Any]		      =    word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)		)
        __lowercase						:     str		      =    char
    return pairs





class   __lowerCAmelCase		(		lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      VOCAB_FILES_NAMES
      A__		:     str       				=      PRETRAINED_VOCAB_FILES_MAP
      A__		:     Dict       				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A__		:     str       				=      ['''input_ids''', '''attention_mask''']




      def __init__( self						:    Union[str, Any]			,    _snake_case						:    Optional[int]			,    _snake_case						:    List[Any]			,    _snake_case						:    Any="replace"			,    _snake_case						:    Union[str, Any]="<s>"			,    _snake_case						:    List[Any]="</s>"			,    _snake_case						:    Any="</s>"			,    _snake_case						:    int="<s>"			,    _snake_case						:    str="<unk>"			,    _snake_case						:    Optional[int]="<pad>"			,    _snake_case						:    Optional[Any]="<mask>"			,    _snake_case						:    Optional[int]=False			,    **_snake_case						:    Any			,    ):
          __lowercase						:     Dict		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else bos_token
          __lowercase						:     int		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else eos_token
          __lowercase						:     Dict		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else sep_token
          __lowercase						:     Tuple		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else cls_token
          __lowercase						:     List[str]		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else unk_token
          __lowercase						:     Tuple		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          __lowercase						:     Dict		      =    AddedToken(_snake_case			,    lstrip=_snake_case			,    rstrip=_snake_case			) if isinstance(_snake_case			,    _snake_case			) else mask_token

          super().__init__(
              errors=_snake_case			,    bos_token=_snake_case			,    eos_token=_snake_case			,    unk_token=_snake_case			,    sep_token=_snake_case			,    cls_token=_snake_case			,    pad_token=_snake_case			,    mask_token=_snake_case			,    add_prefix_space=_snake_case			,    **_snake_case			,    )

          with open(_snake_case			,    encoding='''utf-8'''			) as vocab_handle:
              __lowercase						:     Union[str, Any]		      =    json.load(_snake_case			)
          __lowercase						:     Tuple		      =    {v: k for k, v in self.encoder.items()}
          __lowercase						:     Tuple		      =    errors  # how to handle errors in decoding
          __lowercase						:     int		      =    bytes_to_unicode()
          __lowercase						:     Optional[Any]		      =    {v: k for k, v in self.byte_encoder.items()}
          with open(_snake_case			,    encoding='''utf-8'''			) as merges_handle:
              __lowercase						:     Union[str, Any]		      =    merges_handle.read().split('''\n'''			)[1:-1]
          __lowercase						:     str		      =    [tuple(merge.split()			) for merge in bpe_merges]
          __lowercase						:     List[Any]		      =    dict(zip(_snake_case			,    range(len(_snake_case			)			)			)			)
          __lowercase						:     Optional[int]		      =    {}
          __lowercase						:     Any		      =    add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          __lowercase						:     int		      =    re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''			)




      @property
      def 						snake_case_     ( self						:    Union[str, Any]			):
          return len(self.encoder			)




      def 						snake_case_     ( self						:    List[Any]			):
          return dict(self.encoder			,    **self.added_tokens_encoder			)




      def 						snake_case_     ( self						:    Dict			,    _snake_case						:    List[Any]			):
          if token in self.cache:
              return self.cache[token]
          __lowercase						:     str		      =    tuple(_snake_case			)
          __lowercase						:     int		      =    get_pairs(_snake_case			)

          if not pairs:
              return token

          while True:
              __lowercase						:     str		      =    min(_snake_case			,    key=lambda _snake_case			: self.bpe_ranks.get(_snake_case			,    float('''inf'''			)			)			)
              if bigram not in self.bpe_ranks:
                  break
              __lowercase      , __lowercase						:     List[Any]		      =    bigram
              __lowercase						:     Any		      =    []
              __lowercase						:     List[Any]		      =    0
              while i < len(_snake_case			):
                  try:
                      __lowercase						:     Dict		      =    word.index(_snake_case			,    _snake_case			)
                  except ValueError:
                      new_word.extend(word[i:]			)
                      break
                  else:
                      new_word.extend(word[i:j]			)
                      __lowercase						:     int		      =    j

                  if word[i] == first and i < len(_snake_case			) - 1 and word[i + 1] == second:
                      new_word.append(first + second			)
                      i += 2
                  else:
                      new_word.append(word[i]			)
                      i += 1
              __lowercase						:     int		      =    tuple(_snake_case			)
              __lowercase						:     Union[str, Any]		      =    new_word
              if len(_snake_case			) == 1:
                  break
              else:
                  __lowercase						:     Any		      =    get_pairs(_snake_case			)
          __lowercase						:     Optional[Any]		      =    ''' '''.join(_snake_case			)
          __lowercase						:     str		      =    word
          return word




      def 						snake_case_     ( self						:    List[Any]			,    _snake_case						:    Union[str, Any]			):
          __lowercase						:     str		      =    []
          for token in re.findall(self.pat			,    _snake_case			):
              __lowercase						:     Optional[int]		      =    ''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8'''			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case			).split(''' '''			)			)
          return bpe_tokens




      def 						snake_case_     ( self						:    Optional[Any]			,    _snake_case						:    str			):
          return self.encoder.get(_snake_case			,    self.encoder.get(self.unk_token			)			)




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    List[Any]			):
          return self.decoder.get(_snake_case			)




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    Optional[Any]			):
          __lowercase						:     int		      =    ''''''.join(_snake_case			)
          __lowercase						:     Tuple		      =    bytearray([self.byte_decoder[c] for c in text]			).decode('''utf-8'''			,    errors=self.errors			)
          return text




      def 						snake_case_     ( self						:    Dict			,    _snake_case						:    str			,    _snake_case						:    Optional[str] = None			):
          if not os.path.isdir(_snake_case			):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
              return
          __lowercase						:     Tuple		      =    os.path.join(
              _snake_case			,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
          __lowercase						:     Optional[Any]		      =    os.path.join(
              _snake_case			,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

          with open(_snake_case			,    '''w'''			,    encoding='''utf-8'''			) as f:
              f.write(json.dumps(self.encoder			,    indent=2			,    sort_keys=_snake_case			,    ensure_ascii=_snake_case			) + '''\n'''			)

          __lowercase						:     List[str]		      =    0
          with open(_snake_case			,    '''w'''			,    encoding='''utf-8'''			) as writer:
              writer.write('''#version: 0.2\n'''			)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,    key=lambda _snake_case			: kv[1]			):
                  if index != token_index:
                      logger.warning(
                          F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                          ''' Please check that the tokenizer is not corrupted!'''			)
                      __lowercase						:     int		      =    token_index
                  writer.write(''' '''.join(_snake_case			) + '''\n'''			)
                  index += 1

          return vocab_file, merge_file




      def 						snake_case_     ( self						:    str			,    _snake_case						:    List[int]			,    _snake_case						:    Optional[List[int]] = None			):
          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __lowercase						:     Optional[int]		      =    [self.cls_token_id]
          __lowercase						:     str		      =    [self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep




      def 						snake_case_     ( self						:    Tuple			,    _snake_case						:    List[int]			,    _snake_case						:    Optional[List[int]] = None			,    _snake_case						:    bool = False			):
          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_snake_case			,    token_ids_a=_snake_case			,    already_has_special_tokens=_snake_case			)

          if token_ids_a is None:
              return [1] + ([0] * len(_snake_case			)) + [1]
          return [1] + ([0] * len(_snake_case			)) + [1, 1] + ([0] * len(_snake_case			)) + [1]




      def 						snake_case_     ( self						:    Optional[Any]			,    _snake_case						:    List[int]			,    _snake_case						:    Optional[List[int]] = None			):
          __lowercase						:     Tuple		      =    [self.sep_token_id]
          __lowercase						:     List[Any]		      =    [self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep			) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    List[str]			,    _snake_case						:    Union[str, Any]=False			,    **_snake_case						:    int			):
          __lowercase						:     Tuple		      =    kwargs.pop('''add_prefix_space'''			,    self.add_prefix_space			)
          if (is_split_into_words or add_prefix_space) and (len(_snake_case			) > 0 and not text[0].isspace()):
              __lowercase						:     int		      =    ''' ''' + text
          return (text, kwargs)




from ..utils import DummyObject, requires_backends


class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    List[str]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Tuple			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    int			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    List[Any]			,    *_snake_case						:    List[str]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    int			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Tuple       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Any			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    List[str]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Union[str, Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    List[str]			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Any       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Union[str, Any]			,    *_snake_case						:    Any			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Any			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    str			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Dict			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    str			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[Any]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    str			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Any			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     int       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    List[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    str			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Any			,    *_snake_case						:    str			,    **_snake_case						:    Tuple			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    List[Any]			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[int]       				=      ['''tokenizers''']




      def __init__( self						:    Optional[Any]			,    *_snake_case						:    Dict			,    **_snake_case						:    Optional[int]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Dict			,    **_snake_case						:    int			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     str       				=      ['''tokenizers''']




      def __init__( self						:    Optional[int]			,    *_snake_case						:    List[str]			,    **_snake_case						:    str			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     List[str]       				=      ['''tokenizers''']




      def __init__( self						:    int			,    *_snake_case						:    str			,    **_snake_case						:    Union[str, Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Tuple			,    *_snake_case						:    Optional[Any]			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Dict       				=      ['''tokenizers''']




      def __init__( self						:    Dict			,    *_snake_case						:    Tuple			,    **_snake_case						:    Optional[Any]			):
          requires_backends(self			,    ['''tokenizers''']			)






class   __lowerCAmelCase		(		metaclass=lowerCAmelCase_ ):



      """simple docstring"""







      A__		:     Optional[Any]       				=      ['''tokenizers''']




      def __init__( self						:    List[str]			,    *_snake_case						:    Optional[int]			,    **_snake_case						:    List[str]			):
          requires_backends(self			,    ['''tokenizers''']			)


