





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


_lowerCamelCase							=     logging.get_logger(__name__)

_lowerCamelCase							=     {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase							=     {
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json',
    },
}




class 					UpperCamelCase_    (       UpperCamelCase__      ):


							def __init__(							self      :int							,			__A      :Tuple=None							,			__A      :str=None							,			__A      :List[Any]=False							,			__A      :Dict=True							,			__A      :Optional[Any]=False							,			__A      :Any="<s>"							,			__A      :Dict="</s>"							,			__A      :List[str]="<unk>"							,			__A      :Optional[Any]="<sep>"							,			__A      :str="<pad>"							,			__A      :Optional[Any]="<cls>"							,			__A      :Dict="<mask>"							,			__A      :Dict=["<eop>", "<eod>"]							,			**__A      :str							,			)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= AddedToken(__A							,			lstrip=__A							,			rstrip=__A						) if isinstance(__A							,			__A						) else mask_token

									super().__init__(
									    vocab_file=__A							,			tokenizer_file=__A							,			do_lower_case=__A							,			remove_space=__A							,			keep_accents=__A							,			bos_token=__A							,			eos_token=__A							,			unk_token=__A							,			sep_token=__A							,			pad_token=__A							,			cls_token=__A							,			mask_token=__A							,			additional_special_tokens=__A							,			**__A							,			)

									SCREAMING_SNAKE_CASE__								= 3
									SCREAMING_SNAKE_CASE__								= do_lower_case
									SCREAMING_SNAKE_CASE__								= remove_space
									SCREAMING_SNAKE_CASE__								= keep_accents
									SCREAMING_SNAKE_CASE__								= vocab_file
									SCREAMING_SNAKE_CASE__								= False if not self.vocab_file else True

									try:
											import jieba
									except ModuleNotFoundError as error:
											raise error.__class__(
											    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
											    """See https://pypi.org/project/jieba/ for installation."""						)
									SCREAMING_SNAKE_CASE__								= jieba
									SCREAMING_SNAKE_CASE__								= str.maketrans(""" \n"""							,			"""\u2582\u2583"""						)


							def        _snake_case					(							self      :Optional[Any]							,			__A      :List[int]							,			__A      :Optional[List[int]] = None						)				->				List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								= [self.cls_token_id]
									if token_ids_a is None:
											return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls


							def        _snake_case					(							self      :Dict							,			__A      :List[int]							,			__A      :Optional[List[int]] = None						)				->				List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								= [2]

									if token_ids_a is None:
											return len(token_ids_a + sep						) * [0] + cls_segment_id
									return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id


							def        _snake_case					(							self      :Any							,			__A      :str							,			__A      :Optional[str] = None						)				->				Tuple[str]:



									"""simple docstring"""



									if not self.can_save_slow_tokenizer:
											raise ValueError(
											    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
											    """tokenizer."""						)

									if not os.path.isdir(__A						):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
									SCREAMING_SNAKE_CASE__								= os.path.join(
									    __A							,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(__A						):
											copyfile(self.vocab_file							,			__A						)

									return (out_vocab_file,)


							def        _snake_case					(							self      :str							,			__A      :List[str]							,			*__A      :Optional[int]							,			**__A      :Optional[int]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [
									    """ """.join([x.translate(self.translator						) for x in self.jieba.cut(__A							,			cut_all=__A						)]						)
									    for text in batch_text_or_text_pairs
									]
									return super()._batch_encode_plus(__A							,			*__A							,			**__A						)


							def        _snake_case					(							self      :int							,			*__A      :List[str]							,			**__A      :int						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= super()._decode(*__A							,			**__A						)
									SCREAMING_SNAKE_CASE__								= text.replace(""" """							,			""""""						).replace("""\u2582"""							,			""" """						).replace("""\u2583"""							,			"""\n"""						)
									return text






from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


_lowerCamelCase							=     {
    'configuration_utils': ['GenerationConfig'],
    'streamers': ['TextIteratorStreamer', 'TextStreamer'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase							=     [
	    'Constraint',
	    'ConstraintListState',
	    'DisjunctiveConstraint',
	    'PhrasalConstraint',
	]
	_lowerCamelCase							=     [
	    'BeamHypotheses',
	    'BeamScorer',
	    'BeamSearchScorer',
	    'ConstrainedBeamSearchScorer',
	]
	_lowerCamelCase							=     [
	    'EpsilonLogitsWarper',
	    'EtaLogitsWarper',
	    'ForcedBOSTokenLogitsProcessor',
	    'ForcedEOSTokenLogitsProcessor',
	    'HammingDiversityLogitsProcessor',
	    'InfNanRemoveLogitsProcessor',
	    'LogitsProcessor',
	    'LogitsProcessorList',
	    'LogitsWarper',
	    'MinLengthLogitsProcessor',
	    'MinNewTokensLengthLogitsProcessor',
	    'NoBadWordsLogitsProcessor',
	    'NoRepeatNGramLogitsProcessor',
	    'PrefixConstrainedLogitsProcessor',
	    'RepetitionPenaltyLogitsProcessor',
	    'SequenceBiasLogitsProcessor',
	    'EncoderRepetitionPenaltyLogitsProcessor',
	    'TemperatureLogitsWarper',
	    'TopKLogitsWarper',
	    'TopPLogitsWarper',
	    'TypicalLogitsWarper',
	    'EncoderNoRepeatNGramLogitsProcessor',
	    'ExponentialDecayLengthPenalty',
	    'LogitNormalization',
	]
	_lowerCamelCase							=     [
	    'MaxNewTokensCriteria',
	    'MaxLengthCriteria',
	    'MaxTimeCriteria',
	    'StoppingCriteria',
	    'StoppingCriteriaList',
	    'validate_stopping_criteria',
	]
	_lowerCamelCase							=     [
	    'GenerationMixin',
	    'top_k_top_p_filtering',
	    'GreedySearchEncoderDecoderOutput',
	    'GreedySearchDecoderOnlyOutput',
	    'SampleEncoderDecoderOutput',
	    'SampleDecoderOnlyOutput',
	    'BeamSearchEncoderDecoderOutput',
	    'BeamSearchDecoderOnlyOutput',
	    'BeamSampleEncoderDecoderOutput',
	    'BeamSampleDecoderOnlyOutput',
	    'ContrastiveSearchEncoderDecoderOutput',
	    'ContrastiveSearchDecoderOnlyOutput',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase							=     [
	    'TFForcedBOSTokenLogitsProcessor',
	    'TFForcedEOSTokenLogitsProcessor',
	    'TFLogitsProcessor',
	    'TFLogitsProcessorList',
	    'TFLogitsWarper',
	    'TFMinLengthLogitsProcessor',
	    'TFNoBadWordsLogitsProcessor',
	    'TFNoRepeatNGramLogitsProcessor',
	    'TFRepetitionPenaltyLogitsProcessor',
	    'TFTemperatureLogitsWarper',
	    'TFTopKLogitsWarper',
	    'TFTopPLogitsWarper',
	    'TFForceTokensLogitsProcessor',
	    'TFSuppressTokensAtBeginLogitsProcessor',
	    'TFSuppressTokensLogitsProcessor',
	]
	_lowerCamelCase							=     [
	    'TFGenerationMixin',
	    'tf_top_k_top_p_filtering',
	    'TFGreedySearchDecoderOnlyOutput',
	    'TFGreedySearchEncoderDecoderOutput',
	    'TFSampleEncoderDecoderOutput',
	    'TFSampleDecoderOnlyOutput',
	    'TFBeamSearchEncoderDecoderOutput',
	    'TFBeamSearchDecoderOnlyOutput',
	    'TFBeamSampleEncoderDecoderOutput',
	    'TFBeamSampleDecoderOnlyOutput',
	    'TFContrastiveSearchEncoderDecoderOutput',
	    'TFContrastiveSearchDecoderOnlyOutput',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase							=     [
	    'FlaxForcedBOSTokenLogitsProcessor',
	    'FlaxForcedEOSTokenLogitsProcessor',
	    'FlaxLogitsProcessor',
	    'FlaxLogitsProcessorList',
	    'FlaxLogitsWarper',
	    'FlaxMinLengthLogitsProcessor',
	    'FlaxTemperatureLogitsWarper',
	    'FlaxTopKLogitsWarper',
	    'FlaxTopPLogitsWarper',
	]
	_lowerCamelCase							=     [
	    'FlaxGenerationMixin',
	    'FlaxGreedySearchOutput',
	    'FlaxSampleOutput',
	    'FlaxBeamSearchOutput',
	]

if TYPE_CHECKING:
	from .configuration_utils import GenerationConfig
	from .streamers import TextIteratorStreamer, TextStreamer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
		from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
		from .logits_process import (
		    EncoderNoRepeatNGramLogitsProcessor,
		    EncoderRepetitionPenaltyLogitsProcessor,
		    EpsilonLogitsWarper,
		    EtaLogitsWarper,
		    ExponentialDecayLengthPenalty,
		    ForcedBOSTokenLogitsProcessor,
		    ForcedEOSTokenLogitsProcessor,
		    HammingDiversityLogitsProcessor,
		    InfNanRemoveLogitsProcessor,
		    LogitNormalization,
		    LogitsProcessor,
		    LogitsProcessorList,
		    LogitsWarper,
		    MinLengthLogitsProcessor,
		    MinNewTokensLengthLogitsProcessor,
		    NoBadWordsLogitsProcessor,
		    NoRepeatNGramLogitsProcessor,
		    PrefixConstrainedLogitsProcessor,
		    RepetitionPenaltyLogitsProcessor,
		    SequenceBiasLogitsProcessor,
		    TemperatureLogitsWarper,
		    TopKLogitsWarper,
		    TopPLogitsWarper,
		    TypicalLogitsWarper,
		)
		from .stopping_criteria import (
		    MaxLengthCriteria,
		    MaxNewTokensCriteria,
		    MaxTimeCriteria,
		    StoppingCriteria,
		    StoppingCriteriaList,
		    validate_stopping_criteria,
		)
		from .utils import (
		    BeamSampleDecoderOnlyOutput,
		    BeamSampleEncoderDecoderOutput,
		    BeamSearchDecoderOnlyOutput,
		    BeamSearchEncoderDecoderOutput,
		    ContrastiveSearchDecoderOnlyOutput,
		    ContrastiveSearchEncoderDecoderOutput,
		    GenerationMixin,
		    GreedySearchDecoderOnlyOutput,
		    GreedySearchEncoderDecoderOutput,
		    SampleDecoderOnlyOutput,
		    SampleEncoderDecoderOutput,
		    top_k_top_p_filtering,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tf_logits_process import (
		    TFForcedBOSTokenLogitsProcessor,
		    TFForcedEOSTokenLogitsProcessor,
		    TFForceTokensLogitsProcessor,
		    TFLogitsProcessor,
		    TFLogitsProcessorList,
		    TFLogitsWarper,
		    TFMinLengthLogitsProcessor,
		    TFNoBadWordsLogitsProcessor,
		    TFNoRepeatNGramLogitsProcessor,
		    TFRepetitionPenaltyLogitsProcessor,
		    TFSuppressTokensAtBeginLogitsProcessor,
		    TFSuppressTokensLogitsProcessor,
		    TFTemperatureLogitsWarper,
		    TFTopKLogitsWarper,
		    TFTopPLogitsWarper,
		)
		from .tf_utils import (
		    TFBeamSampleDecoderOnlyOutput,
		    TFBeamSampleEncoderDecoderOutput,
		    TFBeamSearchDecoderOnlyOutput,
		    TFBeamSearchEncoderDecoderOutput,
		    TFContrastiveSearchDecoderOnlyOutput,
		    TFContrastiveSearchEncoderDecoderOutput,
		    TFGenerationMixin,
		    TFGreedySearchDecoderOnlyOutput,
		    TFGreedySearchEncoderDecoderOutput,
		    TFSampleDecoderOnlyOutput,
		    TFSampleEncoderDecoderOutput,
		    tf_top_k_top_p_filtering,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .flax_logits_process import (
		    FlaxForcedBOSTokenLogitsProcessor,
		    FlaxForcedEOSTokenLogitsProcessor,
		    FlaxLogitsProcessor,
		    FlaxLogitsProcessorList,
		    FlaxLogitsWarper,
		    FlaxMinLengthLogitsProcessor,
		    FlaxTemperatureLogitsWarper,
		    FlaxTopKLogitsWarper,
		    FlaxTopPLogitsWarper,
		)
		from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
	import sys

	_lowerCamelCase							=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)