



"""simple docstring"""
import enum
import inspect
import warnings
from functools import wraps
from typing import Callable, Optional

from .logging import get_logger


_UpperCAmelCase       =     set()
_UpperCAmelCase       =     get_logger(__name__)





def        __magic_name__			(							lowercase = None	):

   def decorator(lowercase	):
      global _emitted_deprecation_warnings

      if inspect.isclass(lowercase	):
         SCREAMING_SNAKE_CASE_:	Union[str, Any]            =deprecated_class_or_function.__init__
         SCREAMING_SNAKE_CASE_:	Union[str, Any]            =deprecated_class_or_function.__name__
      else:
         SCREAMING_SNAKE_CASE_:	int            =deprecated_class_or_function
         SCREAMING_SNAKE_CASE_:	Any            =deprecated_function.__name__
         # Support deprecating __init__ class method: class name instead
         SCREAMING_SNAKE_CASE_:	int            =name if name != """__init__""" else deprecated_function.__qualname__.split("""."""	)[-2]

      SCREAMING_SNAKE_CASE_:	Union[str, Any]            =(
          f'''{name} is deprecated and will be removed in the next major version of datasets.''' + f''' {help_message}'''
          if help_message
          else """"""
      )

      @wraps(lowercase	)
      def wrapper(*lowercase							,     **lowercase	):
         SCREAMING_SNAKE_CASE_:	Optional[int]            =hash(lowercase	)
         if func_hash not in _emitted_deprecation_warnings:
            warnings.warn(lowercase							,     category=lowercase							,     stacklevel=2	)
            _emitted_deprecation_warnings.add(lowercase	)
         return deprecated_function(*lowercase							,     **lowercase	)

      SCREAMING_SNAKE_CASE_:	Any            ="""deprecated"""

      if inspect.isclass(lowercase	):
         SCREAMING_SNAKE_CASE_:	Union[str, Any]            =wrapper
         return deprecated_class_or_function
      else:
         return wrapper

   return decorator
class    a		(					enum.EnumMeta    ):



  def __getattribute__(  cls	:   Optional[Any]    ,						lowerCAmelCase	:   str						)       ->   List[str]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[Any]            =super().__getattribute__(lowerCAmelCase						)
     if isinstance(lowerCAmelCase    ,						enum.Enum						) and obj._on_access:
        obj._on_access()
     return obj



  def __getitem__(  cls	:   Optional[int]    ,						lowerCAmelCase	:   int						)       ->   Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	List[str]            =super().__getitem__(lowerCAmelCase						)
     if member._on_access:
        member._on_access()
     return member





  def __call__(  cls	:   List[Any]    ,						lowerCAmelCase	:   int    ,						lowerCAmelCase	:   List[Any]=None    ,						*, lowerCAmelCase	:   Dict=None    ,						lowerCAmelCase	:   List[Any]=None    ,						lowerCAmelCase	:   List[str]=None    ,						lowerCAmelCase	:   Optional[Any]=1						)       ->   Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	List[str]            =super().__call__(lowerCAmelCase    ,						lowerCAmelCase    ,						module=lowerCAmelCase    ,						qualname=lowerCAmelCase    ,						type=lowerCAmelCase    ,						start=lowerCAmelCase						)
     if isinstance(lowerCAmelCase    ,						enum.Enum						) and obj._on_access:
        obj._on_access()
     return obj







class    a		(					enum.Enum		, metaclass=UpperCAmelCase__    ):



  def __new__(  cls	:   List[Any]    ,						lowerCAmelCase	:   List[Any]						)       ->   Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[int]            =object.__new__(cls						)
     SCREAMING_SNAKE_CASE_:	Dict            =value
     SCREAMING_SNAKE_CASE_:	Tuple            =member.deprecate
     return member



  @property
  def 		lowerCamelCase__    (  self	:   Union[str, Any]						)       ->   Optional[int]:


     '''simple docstring'''




     return ""





  def 		lowerCamelCase__    (  self	:   Dict						)       ->   str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	List[Any]            =f''' {self.help_message}''' if self.help_message else """"""
     warnings.warn(
         f'''\'{self.__objclass__.__name__}\' is deprecated and will be removed in the next major version of datasets.'''
         + help_message    ,						lowerCAmelCase    ,						stacklevel=3    ,						)








"""simple docstring"""
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase       =     logging.get_logger(__name__)


_UpperCAmelCase       =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
_UpperCAmelCase       =     {
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

_UpperCAmelCase       =     {
    """RUCAIBox/mvp""": 1_0_2_4,
}





@lru_cache()
def        __magic_name__			(							):
   SCREAMING_SNAKE_CASE_:	Optional[Any]            =(
       list(range(ord("""!"""	)							,     ord("""~"""	) + 1	)	) + list(range(ord("""¡"""	)							,     ord("""¬"""	) + 1	)	) + list(range(ord("""®"""	)							,     ord("""ÿ"""	) + 1	)	)
   )
   SCREAMING_SNAKE_CASE_:	Any            =bs[:]
   SCREAMING_SNAKE_CASE_:	str            =0
   for b in range(2**8	):
      if b not in bs:
         bs.append(lowercase	)
         cs.append(2**8 + n	)
         n += 1
   SCREAMING_SNAKE_CASE_:	List[Any]            =[chr(lowercase	) for n in cs]
   return dict(zip(lowercase							,     lowercase	)	)





def        __magic_name__			(							lowercase	):
   SCREAMING_SNAKE_CASE_:	Optional[int]            =set()
   SCREAMING_SNAKE_CASE_:	Tuple            =word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)	)
      SCREAMING_SNAKE_CASE_:	Dict            =char
   return pairs
class    a		(					UpperCAmelCase__    ):
  UpperCamelCase		:			str         =   VOCAB_FILES_NAMES
  UpperCamelCase		:			List[Any]         =   PRETRAINED_VOCAB_FILES_MAP
  UpperCamelCase		:			Optional[Any]         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCamelCase		:			Optional[int]         =   ['input_ids', 'attention_mask']



  def __init__(  self	:   Tuple    ,						lowerCAmelCase	:   Dict    ,						lowerCAmelCase	:   str    ,						lowerCAmelCase	:   List[str]="replace"    ,						lowerCAmelCase	:   List[Any]="<s>"    ,						lowerCAmelCase	:   str="</s>"    ,						lowerCAmelCase	:   Dict="</s>"    ,						lowerCAmelCase	:   Optional[int]="<s>"    ,						lowerCAmelCase	:   int="<unk>"    ,						lowerCAmelCase	:   Optional[int]="<pad>"    ,						lowerCAmelCase	:   str="<mask>"    ,						lowerCAmelCase	:   Dict=False    ,						**lowerCAmelCase	:   Any    ,						)       ->   Dict:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Any            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else bos_token
     SCREAMING_SNAKE_CASE_:	int            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else eos_token
     SCREAMING_SNAKE_CASE_:	int            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else sep_token
     SCREAMING_SNAKE_CASE_:	Dict            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else cls_token
     SCREAMING_SNAKE_CASE_:	str            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else unk_token
     SCREAMING_SNAKE_CASE_:	int            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     SCREAMING_SNAKE_CASE_:	Optional[int]            =AddedToken(lowerCAmelCase    ,						lstrip=lowerCAmelCase    ,						rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase    ,						lowerCAmelCase						) else mask_token

     super().__init__(
         errors=lowerCAmelCase    ,						bos_token=lowerCAmelCase    ,						eos_token=lowerCAmelCase    ,						unk_token=lowerCAmelCase    ,						sep_token=lowerCAmelCase    ,						cls_token=lowerCAmelCase    ,						pad_token=lowerCAmelCase    ,						mask_token=lowerCAmelCase    ,						add_prefix_space=lowerCAmelCase    ,						**lowerCAmelCase    ,						)
     with open(lowerCAmelCase    ,						encoding="""utf-8"""						) as vocab_handle:
        SCREAMING_SNAKE_CASE_:	Any            =json.load(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Dict            ={v: k for k, v in self.encoder.items()}
     SCREAMING_SNAKE_CASE_:	Optional[Any]            =errors  # how to handle errors in decoding
     SCREAMING_SNAKE_CASE_:	Any            =bytes_to_unicode()
     SCREAMING_SNAKE_CASE_:	str            ={v: k for k, v in self.byte_encoder.items()}
     with open(lowerCAmelCase    ,						encoding="""utf-8"""						) as merges_handle:
        SCREAMING_SNAKE_CASE_:	Union[str, Any]            =merges_handle.read().split("""\n"""						)[1:-1]
     SCREAMING_SNAKE_CASE_:	Dict            =[tuple(merge.split()						) for merge in bpe_merges]
     SCREAMING_SNAKE_CASE_:	Any            =dict(zip(lowerCAmelCase    ,						range(len(lowerCAmelCase						)						)						)						)
     SCREAMING_SNAKE_CASE_:	int            ={}
     SCREAMING_SNAKE_CASE_:	Optional[int]            =add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     SCREAMING_SNAKE_CASE_:	Tuple            =re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)



  @property
  def 		lowerCamelCase__    (  self	:   int						)       ->   List[Any]:


     '''simple docstring'''




     return len(self.encoder						)



  def 		lowerCamelCase__    (  self	:   Dict						)       ->   Optional[Any]:


     '''simple docstring'''




     return dict(self.encoder    ,						**self.added_tokens_encoder						)



  def 		lowerCamelCase__    (  self	:   List[str]    ,						lowerCAmelCase	:   Union[str, Any]						)       ->   Any:


     '''simple docstring'''




     if token in self.cache:
        return self.cache[token]
     SCREAMING_SNAKE_CASE_:	Optional[Any]            =tuple(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =get_pairs(lowerCAmelCase						)

     if not pairs:
        return token

     while True:
        SCREAMING_SNAKE_CASE_:	int            =min(lowerCAmelCase    ,						key=lambda lowerCAmelCase						: self.bpe_ranks.get(lowerCAmelCase    ,						float("""inf"""						)						)						)
        if bigram not in self.bpe_ranks:
           break
        SCREAMING_SNAKE_CASE_     ,						SCREAMING_SNAKE_CASE_:	Optional[Any]            =bigram
        SCREAMING_SNAKE_CASE_:	List[Any]            =[]
        SCREAMING_SNAKE_CASE_:	int            =0
        while i < len(lowerCAmelCase						):
           try:
              SCREAMING_SNAKE_CASE_:	List[Any]            =word.index(lowerCAmelCase    ,						lowerCAmelCase						)
           except ValueError:
              new_word.extend(word[i:]						)
              break
           else:
              new_word.extend(word[i:j]						)
              SCREAMING_SNAKE_CASE_:	str            =j

           if word[i] == first and i < len(lowerCAmelCase						) - 1 and word[i + 1] == second:
              new_word.append(first + second						)
              i += 2
           else:
              new_word.append(word[i]						)
              i += 1
        SCREAMING_SNAKE_CASE_:	Any            =tuple(lowerCAmelCase						)
        SCREAMING_SNAKE_CASE_:	str            =new_word
        if len(lowerCAmelCase						) == 1:
           break
        else:
           SCREAMING_SNAKE_CASE_:	List[str]            =get_pairs(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Optional[Any]            =""" """.join(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	List[Any]            =word
     return word



  def 		lowerCamelCase__    (  self	:   Optional[int]    ,						lowerCAmelCase	:   Optional[int]						)       ->   Tuple:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Tuple            =[]
     for token in re.findall(self.pat    ,						lowerCAmelCase						):
        SCREAMING_SNAKE_CASE_:	Union[str, Any]            ="""""".join(
            self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase						).split(""" """						)						)
     return bpe_tokens



  def 		lowerCamelCase__    (  self	:   List[str]    ,						lowerCAmelCase	:   str						)       ->   str:


     '''simple docstring'''




     return self.encoder.get(lowerCAmelCase    ,						self.encoder.get(self.unk_token						)						)



  def 		lowerCamelCase__    (  self	:   Any    ,						lowerCAmelCase	:   Optional[Any]						)       ->   Optional[int]:


     '''simple docstring'''




     return self.decoder.get(lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   Dict    ,						lowerCAmelCase	:   Optional[Any]						)       ->   Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[int]            ="""""".join(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Dict            =bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""    ,						errors=self.errors						)
     return text



  def 		lowerCamelCase__    (  self	:   Optional[Any]    ,						lowerCAmelCase	:   str    ,						lowerCAmelCase	:   Optional[str] = None						)       ->   Tuple[str]:


     '''simple docstring'''




     if not os.path.isdir(lowerCAmelCase						):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
        return
     SCREAMING_SNAKE_CASE_:	Any            =os.path.join(
         lowerCAmelCase    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
     SCREAMING_SNAKE_CASE_:	List[Any]            =os.path.join(
         lowerCAmelCase    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

     with open(lowerCAmelCase    ,						"""w"""    ,						encoding="""utf-8"""						) as f:
        f.write(json.dumps(self.encoder    ,						indent=2    ,						sort_keys=lowerCAmelCase    ,						ensure_ascii=lowerCAmelCase						) + """\n"""						)

     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =0
     with open(lowerCAmelCase    ,						"""w"""    ,						encoding="""utf-8"""						) as writer:
        writer.write("""#version: 0.2\n"""						)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,						key=lambda lowerCAmelCase						: kv[1]						):
           if index != token_index:
              logger.warning(
                  f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                  """ Please check that the tokenizer is not corrupted!"""						)
              SCREAMING_SNAKE_CASE_:	Union[str, Any]            =token_index
           writer.write(""" """.join(lowerCAmelCase						) + """\n"""						)
           index += 1

     return vocab_file, merge_file



  def 		lowerCamelCase__    (  self	:   Any    ,						lowerCAmelCase	:   List[int]    ,						lowerCAmelCase	:   Optional[List[int]] = None						)       ->   List[int]:


     '''simple docstring'''




     if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     SCREAMING_SNAKE_CASE_:	List[str]            =[self.cls_token_id]
     SCREAMING_SNAKE_CASE_:	List[str]            =[self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep



  def 		lowerCamelCase__    (  self	:   List[str]    ,						lowerCAmelCase	:   List[int]    ,						lowerCAmelCase	:   Optional[List[int]] = None    ,						lowerCAmelCase	:   bool = False						)       ->   List[int]:


     '''simple docstring'''




     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCAmelCase    ,						token_ids_a=lowerCAmelCase    ,						already_has_special_tokens=lowerCAmelCase						)

     if token_ids_a is None:
        return [1] + ([0] * len(lowerCAmelCase						)) + [1]
     return [1] + ([0] * len(lowerCAmelCase						)) + [1, 1] + ([0] * len(lowerCAmelCase						)) + [1]



  def 		lowerCamelCase__    (  self	:   Dict    ,						lowerCAmelCase	:   List[int]    ,						lowerCAmelCase	:   Optional[List[int]] = None						)       ->   List[int]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[Any]            =[self.sep_token_id]
     SCREAMING_SNAKE_CASE_:	str            =[self.cls_token_id]

     if token_ids_a is None:
        return len(cls + token_ids_a + sep						) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]





  def 		lowerCamelCase__    (  self	:   Optional[int]    ,						lowerCAmelCase	:   Optional[int]    ,						lowerCAmelCase	:   Union[str, Any]=False    ,						**lowerCAmelCase	:   str						)       ->   Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	int            =kwargs.pop("""add_prefix_space"""    ,						self.add_prefix_space						)
     if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase						) > 0 and not text[0].isspace()):
        SCREAMING_SNAKE_CASE_:	List[Any]            =""" """ + text
     return (text, kwargs)



