






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__a :    Dict								=				logging.get_logger(__name__)


__a :    str								=				{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all MVP models at https://huggingface.co/models?filter=mvp
__a :    Union[str, Any]								=				{
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
}

__a :    List[str]								=				{
    "RUCAIBox/mvp": 1024,
}



@lru_cache()
def      _SCREAMING_SNAKE_CASE  (	)  -> str:





    """simple docstring"""

    __A             =		(
        list(range(ord("""!"""      )     ,			ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )     ,			ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )     ,			ord("""ÿ"""      ) + 1      )      )
    )
    __A             =		bs[:]
    __A             =		0
    for b in range(2**8      ):
        if b not in bs:
            bs.append(__lowercase      )
            cs.append(2**8 + n      )
            n += 1
    __A             =		[chr(__lowercase      ) for n in cs]
    return dict(zip(__lowercase     ,			__lowercase      )      )



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Optional[int]      )  -> Optional[Any]:





    """simple docstring"""

    __A             =		set()
    __A             =		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)      )
        __A             =		char
    return pairs






class  __lowercase (		lowercase_					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE				  =			PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE				  =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE				  =			["input_ids", "attention_mask"]




     def __init__(			self    :     Union[str, Any]	,		UpperCamelCase_    :     str	,		UpperCamelCase_    :     str	,		UpperCamelCase_    :     Tuple="replace"	,		UpperCamelCase_    :     Optional[Any]="<s>"	,		UpperCamelCase_    :     Any="</s>"	,		UpperCamelCase_    :     Tuple="</s>"	,		UpperCamelCase_    :     Optional[Any]="<s>"	,		UpperCamelCase_    :     List[Any]="<unk>"	,		UpperCamelCase_    :     Union[str, Any]="<pad>"	,		UpperCamelCase_    :     str="<mask>"	,		UpperCamelCase_    :     Optional[int]=False	,		**UpperCamelCase_    :     Tuple	,		):


         """simple docstring"""



         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else bos_token
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else eos_token
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else sep_token
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else cls_token
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else unk_token
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else mask_token

         super().__init__(
             errors=UpperCamelCase_	,		bos_token=UpperCamelCase_	,		eos_token=UpperCamelCase_	,		unk_token=UpperCamelCase_	,		sep_token=UpperCamelCase_	,		cls_token=UpperCamelCase_	,		pad_token=UpperCamelCase_	,		mask_token=UpperCamelCase_	,		add_prefix_space=UpperCamelCase_	,		**UpperCamelCase_	,		)
         with open(UpperCamelCase_	,		encoding="""utf-8"""			) as vocab_handle:
             __A             =		json.load(UpperCamelCase_			)
         __A             =		{v: k for k, v in self.encoder.items()}
         __A             =		errors  # how to handle errors in decoding
         __A             =		bytes_to_unicode()
         __A             =		{v: k for k, v in self.byte_encoder.items()}
         with open(UpperCamelCase_	,		encoding="""utf-8"""			) as merges_handle:
             __A             =		merges_handle.read().split("""\n"""			)[1:-1]
         __A             =		[tuple(merge.split()			) for merge in bpe_merges]
         __A             =		dict(zip(UpperCamelCase_	,		range(len(UpperCamelCase_			)			)			)			)
         __A             =		{}
         __A             =		add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __A             =		re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)




     @property
     def 						lowerCAmelCase_       (			self    :     List[Any]			):


         """simple docstring"""



         return len(self.encoder			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         return dict(self.encoder	,		**self.added_tokens_encoder			)




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     Optional[Any]			):


         """simple docstring"""



         if token in self.cache:
             return self.cache[token]
         __A             =		tuple(UpperCamelCase_			)
         __A             =		get_pairs(UpperCamelCase_			)

         if not pairs:
             return token

         while True:
             __A             =		min(UpperCamelCase_	,		key=lambda UpperCamelCase_			: self.bpe_ranks.get(UpperCamelCase_	,		float("""inf"""			)			)			)
             if bigram not in self.bpe_ranks:
                 break
             __A      ,		__A             =		bigram
             __A             =		[]
             __A             =		0
             while i < len(UpperCamelCase_			):
                 try:
                     __A             =		word.index(UpperCamelCase_	,		UpperCamelCase_			)
                 except ValueError:
                     new_word.extend(word[i:]			)
                     break
                 else:
                     new_word.extend(word[i:j]			)
                     __A             =		j

                 if word[i] == first and i < len(UpperCamelCase_			) - 1 and word[i + 1] == second:
                     new_word.append(first + second			)
                     i += 2
                 else:
                     new_word.append(word[i]			)
                     i += 1
             __A             =		tuple(UpperCamelCase_			)
             __A             =		new_word
             if len(UpperCamelCase_			) == 1:
                 break
             else:
                 __A             =		get_pairs(UpperCamelCase_			)
         __A             =		""" """.join(UpperCamelCase_			)
         __A             =		word
         return word




     def 						lowerCAmelCase_       (			self    :     Optional[Any]	,		UpperCamelCase_    :     List[Any]			):


         """simple docstring"""



         __A             =		[]
         for token in re.findall(self.pat	,		UpperCamelCase_			):
             __A             =		"""""".join(
                 self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_			).split(""" """			)			)
         return bpe_tokens




     def 						lowerCAmelCase_       (			self    :     Tuple	,		UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         return self.encoder.get(UpperCamelCase_	,		self.encoder.get(self.unk_token			)			)




     def 						lowerCAmelCase_       (			self    :     Any	,		UpperCamelCase_    :     List[Any]			):


         """simple docstring"""



         return self.decoder.get(UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     List[str]	,		UpperCamelCase_    :     int			):


         """simple docstring"""



         __A             =		"""""".join(UpperCamelCase_			)
         __A             =		bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""	,		errors=self.errors			)
         return text




     def 						lowerCAmelCase_       (			self    :     Optional[int]	,		UpperCamelCase_    :     str	,		UpperCamelCase_    :     Optional[str] = None			):


         """simple docstring"""



         if not os.path.isdir(UpperCamelCase_			):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
             return
         __A             =		os.path.join(
             UpperCamelCase_	,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
         __A             =		os.path.join(
             UpperCamelCase_	,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

         with open(UpperCamelCase_	,		"""w"""	,		encoding="""utf-8"""			) as f:
             f.write(json.dumps(self.encoder	,		indent=2	,		sort_keys=UpperCamelCase_	,		ensure_ascii=UpperCamelCase_			) + """\n"""			)

         __A             =		0
         with open(UpperCamelCase_	,		"""w"""	,		encoding="""utf-8"""			) as writer:
             writer.write("""#version: 0.2\n"""			)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,		key=lambda UpperCamelCase_			: kv[1]			):
                 if index != token_index:
                     logger.warning(
                         F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                         """ Please check that the tokenizer is not corrupted!"""			)
                     __A             =		token_index
                 writer.write(""" """.join(UpperCamelCase_			) + """\n"""			)
                 index += 1

         return vocab_file, merge_file




     def 						lowerCAmelCase_       (			self    :     Optional[int]	,		UpperCamelCase_    :     List[int]	,		UpperCamelCase_    :     Optional[List[int]] = None			):


         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __A             =		[self.cls_token_id]
         __A             =		[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




     def 						lowerCAmelCase_       (			self    :     List[str]	,		UpperCamelCase_    :     List[int]	,		UpperCamelCase_    :     Optional[List[int]] = None	,		UpperCamelCase_    :     bool = False			):


         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase_	,		token_ids_a=UpperCamelCase_	,		already_has_special_tokens=UpperCamelCase_			)

         if token_ids_a is None:
             return [1] + ([0] * len(UpperCamelCase_			)) + [1]
         return [1] + ([0] * len(UpperCamelCase_			)) + [1, 1] + ([0] * len(UpperCamelCase_			)) + [1]




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     List[int]	,		UpperCamelCase_    :     Optional[List[int]] = None			):


         """simple docstring"""



         __A             =		[self.sep_token_id]
         __A             =		[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep			) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]





     def 						lowerCAmelCase_       (			self    :     Union[str, Any]	,		UpperCamelCase_    :     Dict	,		UpperCamelCase_    :     Tuple=False	,		**UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         __A             =		kwargs.pop("""add_prefix_space"""	,		self.add_prefix_space			)
         if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_			) > 0 and not text[0].isspace()):
             __A             =		""" """ + text
         return (text, kwargs)












import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__a :    List[str]								=				get_logger(__name__)


if TYPE_CHECKING:
  from .builder import DatasetBuilder



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Dict     ,			__lowercase				:     Optional[Union[str, bool]] = None      )  -> List[str]:





    """simple docstring"""

    __A             =		importlib.import_module(__lowercase      )

    # TODO(QL): always update the module to add subsequent new authentication
    if hasattr(__lowercase     ,			"""_patched_for_streaming"""      ) and module._patched_for_streaming:
        return

    def wrap_auth(__lowercase				:     int      ):
        @wraps(__lowercase      )
        def wrapper(*__lowercase				:     Tuple     ,			**__lowercase				:     List[str]      ):
            return function(*__lowercase     ,			use_auth_token=__lowercase     ,			**__lowercase      )

        __A             =		"""wrap_auth"""
        return wrapper

    # open files in a streaming fashion
    patch_submodule(__lowercase     ,			"""open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.listdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.walk"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""glob.glob"""     ,			wrap_auth(__lowercase      )      ).start()
    # allow to navigate in remote zip files
    patch_submodule(__lowercase     ,			"""os.path.join"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.dirname"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.basename"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.relpath"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.split"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.splitext"""     ,			__lowercase      ).start()
    # allow checks on paths
    patch_submodule(__lowercase     ,			"""os.path.exists"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isfile"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.getsize"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pathlib.Path"""     ,			__lowercase      ).start()
    # file readers
    patch_submodule(__lowercase     ,			"""gzip.open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""numpy.load"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_csv"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_excel"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""scipy.io.loadmat"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""xml.etree.ElementTree.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""xml.dom.minidom.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    __A             =		True






def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     "DatasetBuilder"      )  -> Union[str, Any]:





    """simple docstring"""

    extend_module_for_streaming(builder.__module__     ,			use_auth_token=builder.use_auth_token      )
    # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
    if not builder.__module__.startswith("""datasets."""      ):  # check that it's not a packaged builder like csv
        for imports in get_imports(inspect.getfile(builder.__class__      )      ):
            if imports[0] == "internal":
                __A             =		imports[1]
                __A             =		""".""".join(builder.__module__.split("""."""      )[:-1] + [internal_import_name]      )
                extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
    from .builder import DatasetBuilder

    __A             =		[
        cls.__module__
        for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
        if issubclass(cls     ,			__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
    ]  # check it's not a standard builder from datasets.builder
    for module in parent_builder_modules:
        extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )




