


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


_snake_case					=					{
    '''configuration_utils''': ['''GenerationConfig'''],
    '''streamers''': ['''TextIteratorStreamer''', '''TextStreamer'''],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _snake_case					=					[
       '''Constraint''',
       '''ConstraintListState''',
       '''DisjunctiveConstraint''',
       '''PhrasalConstraint''',
   ]
   _snake_case					=					[
       '''BeamHypotheses''',
       '''BeamScorer''',
       '''BeamSearchScorer''',
       '''ConstrainedBeamSearchScorer''',
   ]
   _snake_case					=					[
       '''EpsilonLogitsWarper''',
       '''EtaLogitsWarper''',
       '''ForcedBOSTokenLogitsProcessor''',
       '''ForcedEOSTokenLogitsProcessor''',
       '''HammingDiversityLogitsProcessor''',
       '''InfNanRemoveLogitsProcessor''',
       '''LogitsProcessor''',
       '''LogitsProcessorList''',
       '''LogitsWarper''',
       '''MinLengthLogitsProcessor''',
       '''MinNewTokensLengthLogitsProcessor''',
       '''NoBadWordsLogitsProcessor''',
       '''NoRepeatNGramLogitsProcessor''',
       '''PrefixConstrainedLogitsProcessor''',
       '''RepetitionPenaltyLogitsProcessor''',
       '''SequenceBiasLogitsProcessor''',
       '''EncoderRepetitionPenaltyLogitsProcessor''',
       '''TemperatureLogitsWarper''',
       '''TopKLogitsWarper''',
       '''TopPLogitsWarper''',
       '''TypicalLogitsWarper''',
       '''EncoderNoRepeatNGramLogitsProcessor''',
       '''ExponentialDecayLengthPenalty''',
       '''LogitNormalization''',
   ]
   _snake_case					=					[
       '''MaxNewTokensCriteria''',
       '''MaxLengthCriteria''',
       '''MaxTimeCriteria''',
       '''StoppingCriteria''',
       '''StoppingCriteriaList''',
       '''validate_stopping_criteria''',
   ]
   _snake_case					=					[
       '''GenerationMixin''',
       '''top_k_top_p_filtering''',
       '''GreedySearchEncoderDecoderOutput''',
       '''GreedySearchDecoderOnlyOutput''',
       '''SampleEncoderDecoderOutput''',
       '''SampleDecoderOnlyOutput''',
       '''BeamSearchEncoderDecoderOutput''',
       '''BeamSearchDecoderOnlyOutput''',
       '''BeamSampleEncoderDecoderOutput''',
       '''BeamSampleDecoderOnlyOutput''',
       '''ContrastiveSearchEncoderDecoderOutput''',
       '''ContrastiveSearchDecoderOnlyOutput''',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _snake_case					=					[
       '''TFForcedBOSTokenLogitsProcessor''',
       '''TFForcedEOSTokenLogitsProcessor''',
       '''TFLogitsProcessor''',
       '''TFLogitsProcessorList''',
       '''TFLogitsWarper''',
       '''TFMinLengthLogitsProcessor''',
       '''TFNoBadWordsLogitsProcessor''',
       '''TFNoRepeatNGramLogitsProcessor''',
       '''TFRepetitionPenaltyLogitsProcessor''',
       '''TFTemperatureLogitsWarper''',
       '''TFTopKLogitsWarper''',
       '''TFTopPLogitsWarper''',
       '''TFForceTokensLogitsProcessor''',
       '''TFSuppressTokensAtBeginLogitsProcessor''',
       '''TFSuppressTokensLogitsProcessor''',
   ]
   _snake_case					=					[
       '''TFGenerationMixin''',
       '''tf_top_k_top_p_filtering''',
       '''TFGreedySearchDecoderOnlyOutput''',
       '''TFGreedySearchEncoderDecoderOutput''',
       '''TFSampleEncoderDecoderOutput''',
       '''TFSampleDecoderOnlyOutput''',
       '''TFBeamSearchEncoderDecoderOutput''',
       '''TFBeamSearchDecoderOnlyOutput''',
       '''TFBeamSampleEncoderDecoderOutput''',
       '''TFBeamSampleDecoderOnlyOutput''',
       '''TFContrastiveSearchEncoderDecoderOutput''',
       '''TFContrastiveSearchDecoderOnlyOutput''',
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _snake_case					=					[
       '''FlaxForcedBOSTokenLogitsProcessor''',
       '''FlaxForcedEOSTokenLogitsProcessor''',
       '''FlaxLogitsProcessor''',
       '''FlaxLogitsProcessorList''',
       '''FlaxLogitsWarper''',
       '''FlaxMinLengthLogitsProcessor''',
       '''FlaxTemperatureLogitsWarper''',
       '''FlaxTopKLogitsWarper''',
       '''FlaxTopPLogitsWarper''',
   ]
   _snake_case					=					[
       '''FlaxGenerationMixin''',
       '''FlaxGreedySearchOutput''',
       '''FlaxSampleOutput''',
       '''FlaxBeamSearchOutput''',
   ]

if TYPE_CHECKING:
   from .configuration_utils import GenerationConfig
   from .streamers import TextIteratorStreamer, TextStreamer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
      from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
      from .logits_process import (
          EncoderNoRepeatNGramLogitsProcessor,
          EncoderRepetitionPenaltyLogitsProcessor,
          EpsilonLogitsWarper,
          EtaLogitsWarper,
          ExponentialDecayLengthPenalty,
          ForcedBOSTokenLogitsProcessor,
          ForcedEOSTokenLogitsProcessor,
          HammingDiversityLogitsProcessor,
          InfNanRemoveLogitsProcessor,
          LogitNormalization,
          LogitsProcessor,
          LogitsProcessorList,
          LogitsWarper,
          MinLengthLogitsProcessor,
          MinNewTokensLengthLogitsProcessor,
          NoBadWordsLogitsProcessor,
          NoRepeatNGramLogitsProcessor,
          PrefixConstrainedLogitsProcessor,
          RepetitionPenaltyLogitsProcessor,
          SequenceBiasLogitsProcessor,
          TemperatureLogitsWarper,
          TopKLogitsWarper,
          TopPLogitsWarper,
          TypicalLogitsWarper,
      )
      from .stopping_criteria import (
          MaxLengthCriteria,
          MaxNewTokensCriteria,
          MaxTimeCriteria,
          StoppingCriteria,
          StoppingCriteriaList,
          validate_stopping_criteria,
      )
      from .utils import (
          BeamSampleDecoderOnlyOutput,
          BeamSampleEncoderDecoderOutput,
          BeamSearchDecoderOnlyOutput,
          BeamSearchEncoderDecoderOutput,
          ContrastiveSearchDecoderOnlyOutput,
          ContrastiveSearchEncoderDecoderOutput,
          GenerationMixin,
          GreedySearchDecoderOnlyOutput,
          GreedySearchEncoderDecoderOutput,
          SampleDecoderOnlyOutput,
          SampleEncoderDecoderOutput,
          top_k_top_p_filtering,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tf_logits_process import (
          TFForcedBOSTokenLogitsProcessor,
          TFForcedEOSTokenLogitsProcessor,
          TFForceTokensLogitsProcessor,
          TFLogitsProcessor,
          TFLogitsProcessorList,
          TFLogitsWarper,
          TFMinLengthLogitsProcessor,
          TFNoBadWordsLogitsProcessor,
          TFNoRepeatNGramLogitsProcessor,
          TFRepetitionPenaltyLogitsProcessor,
          TFSuppressTokensAtBeginLogitsProcessor,
          TFSuppressTokensLogitsProcessor,
          TFTemperatureLogitsWarper,
          TFTopKLogitsWarper,
          TFTopPLogitsWarper,
      )
      from .tf_utils import (
          TFBeamSampleDecoderOnlyOutput,
          TFBeamSampleEncoderDecoderOutput,
          TFBeamSearchDecoderOnlyOutput,
          TFBeamSearchEncoderDecoderOutput,
          TFContrastiveSearchDecoderOnlyOutput,
          TFContrastiveSearchEncoderDecoderOutput,
          TFGenerationMixin,
          TFGreedySearchDecoderOnlyOutput,
          TFGreedySearchEncoderDecoderOutput,
          TFSampleDecoderOnlyOutput,
          TFSampleEncoderDecoderOutput,
          tf_top_k_top_p_filtering,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .flax_logits_process import (
          FlaxForcedBOSTokenLogitsProcessor,
          FlaxForcedEOSTokenLogitsProcessor,
          FlaxLogitsProcessor,
          FlaxLogitsProcessorList,
          FlaxLogitsWarper,
          FlaxMinLengthLogitsProcessor,
          FlaxTemperatureLogitsWarper,
          FlaxTopKLogitsWarper,
          FlaxTopPLogitsWarper,
      )
      from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
   import sys

   _snake_case					=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


_snake_case					=					logging.get_logger(__name__)

_snake_case					=					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_snake_case					=					{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

_snake_case					=					{
    '''microsoft/mpnet-base''': 5_12,
}

_snake_case					=					{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}
class      UpperCAmelCase_ ( UpperCamelCase   ):

   '''simple docstring'''




   __A			:				List[str]							=		VOCAB_FILES_NAMES
   __A			:				int							=		PRETRAINED_VOCAB_FILES_MAP
   __A			:				str							=		PRETRAINED_INIT_CONFIGURATION
   __A			:				int							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __A			:				Any							=		MPNetTokenizer
   __A			:				Dict							=		["input_ids", "attention_mask"]






   def __init__(		self						,    __A=None						,    __A=None						,    __A=True						,    __A="<s>"						,    __A="</s>"						,    __A="</s>"						,    __A="<s>"						,    __A="[UNK]"						,    __A="<pad>"						,    __A="<mask>"						,    __A=True						,    __A=None						,    **__A						,    ):






     """simple docstring"""





     super().__init__(
         __A						,    tokenizer_file=__A						,    do_lower_case=__A						,    bos_token=__A						,    eos_token=__A						,    sep_token=__A						,    cls_token=__A						,    unk_token=__A						,    pad_token=__A						,    mask_token=__A						,    tokenize_chinese_chars=__A						,    strip_accents=__A						,    **__A						,    )

     lowerCamelCase			: List[Any]   =    json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         pre_tok_state.get("lowercase"						,    __A			) != do_lower_case
         or pre_tok_state.get("strip_accents"						,    __A			) != strip_accents
     ):
       lowerCamelCase			: Dict   =    getattr(__A						,    pre_tok_state.pop("type"			)			)
       lowerCamelCase			: List[str]   =    do_lower_case
       lowerCamelCase			: Tuple   =    strip_accents
       lowerCamelCase			: int   =    pre_tok_class(**__A			)

     lowerCamelCase			: List[Any]   =    do_lower_case






   @property
   def _snake_case   (		self			):






     """simple docstring"""





     if self._mask_token is None:
       if self.verbose:
         logger.error("Using mask_token, but it is not set yet."			)
       return None
     return str(self._mask_token			)






   @mask_token.setter
   def _snake_case   (		self						,    __A			):






     """simple docstring"""





     lowerCamelCase			: Tuple   =    AddedToken(__A						,    lstrip=__A						,    rstrip=__A			) if isinstance(__A						,    __A			) else value
     lowerCamelCase			: Dict   =    value






   def _snake_case   (		self						,    __A						,    __A=None			):






     """simple docstring"""





     lowerCamelCase			: Optional[Any]   =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
       return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






   def _snake_case   (		self						,    __A						,    __A = None			):






     """simple docstring"""





     lowerCamelCase			: Any   =    [self.sep_token_id]
     lowerCamelCase			: Union[str, Any]   =    [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






   def _snake_case   (		self						,    __A						,    __A = None			):






     """simple docstring"""





     lowerCamelCase			: int   =    self._tokenizer.model.save(__A						,    name=__A			)
     return tuple(__A			)



