



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


A__     				=	logging.get_logger(__name__)

A__     				=	{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

A__     				=	{
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json""",
    },
}





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case=None						,    _snake_case=None						,    _snake_case=False						,    _snake_case=True						,    _snake_case=False						,    _snake_case="<s>"						,    _snake_case="</s>"						,    _snake_case="<unk>"						,    _snake_case="<sep>"						,    _snake_case="<pad>"						,    _snake_case="<cls>"						,    _snake_case="<mask>"						,    _snake_case=["<eop>", "<eod>"]						,    **_snake_case						,    ):



									"""simple docstring"""





									_lowerCAmelCase      =   AddedToken(_snake_case						,    lstrip=_snake_case						,    rstrip=_snake_case		) if isinstance(_snake_case						,    _snake_case		) else mask_token

									super().__init__(
									    vocab_file=_snake_case						,    tokenizer_file=_snake_case						,    do_lower_case=_snake_case						,    remove_space=_snake_case						,    keep_accents=_snake_case						,    bos_token=_snake_case						,    eos_token=_snake_case						,    unk_token=_snake_case						,    sep_token=_snake_case						,    pad_token=_snake_case						,    cls_token=_snake_case						,    mask_token=_snake_case						,    additional_special_tokens=_snake_case						,    **_snake_case						,    )

									_lowerCAmelCase      =   3
									_lowerCAmelCase      =   do_lower_case
									_lowerCAmelCase      =   remove_space
									_lowerCAmelCase      =   keep_accents
									_lowerCAmelCase      =   vocab_file
									_lowerCAmelCase      =   False if not self.vocab_file else True

									try:
														import jieba
									except ModuleNotFoundError as error:
														raise error.__class__(
														    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
														    """See https://pypi.org/project/jieba/ for installation."""		)
									_lowerCAmelCase      =   jieba
									_lowerCAmelCase      =   str.maketrans(""" \n"""						,    """\u2582\u2583"""		)




				def 							snake_case       (							self						,    _snake_case						,    _snake_case = None		):



									"""simple docstring"""





									_lowerCAmelCase      =   [self.sep_token_id]
									_lowerCAmelCase      =   [self.cls_token_id]
									if token_ids_a is None:
														return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls




				def 							snake_case       (							self						,    _snake_case						,    _snake_case = None		):



									"""simple docstring"""





									_lowerCAmelCase      =   [self.sep_token_id]
									_lowerCAmelCase      =   [2]

									if token_ids_a is None:
														return len(token_ids_a + sep		) * [0] + cls_segment_id
									return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id




				def 							snake_case       (							self						,    _snake_case						,    _snake_case = None		):



									"""simple docstring"""





									if not self.can_save_slow_tokenizer:
														raise ValueError(
														    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
														    """tokenizer."""		)

									if not os.path.isdir(_snake_case		):
														logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
														return
									_lowerCAmelCase      =   os.path.join(
									    _snake_case						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

									if os.path.abspath(self.vocab_file		) != os.path.abspath(_snake_case		):
														copyfile(self.vocab_file						,    _snake_case		)

									return (out_vocab_file,)




				def 							snake_case       (							self						,    _snake_case						,    *_snake_case						,    **_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   [
									    """ """.join([x.translate(self.translator		) for x in self.jieba.cut(_snake_case						,    cut_all=_snake_case		)]		)
									    for text in batch_text_or_text_pairs
									]
									return super()._batch_encode_plus(_snake_case						,    *_snake_case						,    **_snake_case		)




				def 							snake_case       (							self						,    *_snake_case						,    **_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   super()._decode(*_snake_case						,    **_snake_case		)
									_lowerCAmelCase      =   text.replace(""" """						,    """"""		).replace("""\u2582"""						,    """ """		).replace("""\u2583"""						,    """\n"""		)
									return text








# Lint as: python3

import hashlib
from typing import Union


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					if isinstance(snake_case     ,						snake_case      ):
										# Data already in bytes, returns as it as
										return hash_data
					elif isinstance(snake_case     ,						snake_case      ):
										# We keep the data as it as for it ot be later encoded to UTF-8
										# However replace `\\` with `/` for Windows compatibility
										_lowerCAmelCase      =   hash_data.replace("""\\"""     ,						"""/"""      )
					elif isinstance(snake_case     ,						snake_case      ):
										_lowerCAmelCase      =   str(snake_case      )
					else:
										# If data is not of the required type, raise error
										raise InvalidKeyError(snake_case      )

					return hash_data.encode("""utf-8"""      )





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   """\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
									_lowerCAmelCase      =   F'\nFound Key {hash_data} of type {type(_snake_case		)}'
									_lowerCAmelCase      =   """\nKeys should be either str, int or bytes type"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case=""		):



									"""simple docstring"""





									_lowerCAmelCase      =   key
									_lowerCAmelCase      =   duplicate_key_indices
									_lowerCAmelCase      =   fix_msg
									_lowerCAmelCase      =   """Found multiple examples generated with the same key"""
									if len(_snake_case		) <= 20:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(_snake_case		)} have the key {key}'
									else:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(duplicate_key_indices[:20]		)}... ({len(_snake_case		) - 20} more) have the key {key}'
									_lowerCAmelCase      =   """\n""" + fix_msg if fix_msg else """"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   hashlib.mda(_as_bytes(_snake_case		)		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self._split_mda.copy()
									_lowerCAmelCase      =   _as_bytes(_snake_case		)
									mda.update(_snake_case		)
									# Convert to integer with hexadecimal conversion
									return int(mda.hexdigest()						,    16		)



