






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__UpperCAmelCase      				=							logging.get_logger(__name__)

__UpperCAmelCase      				=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase      				=							{
    "vocab_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json",
    },
}

__UpperCAmelCase      				=							{
    "microsoft/mpnet-base": 5_1_2,
}

__UpperCAmelCase      				=							{
    "microsoft/mpnet-base": {"do_lower_case": True},
}
class 					a_(						lowerCAmelCase__					):



     """simple docstring"""



     __snake_case						:							Optional[int]       			=VOCAB_FILES_NAMES
     __snake_case						:							Union[str, Any]       			=PRETRAINED_VOCAB_FILES_MAP
     __snake_case						:							Optional[int]       			=PRETRAINED_INIT_CONFIGURATION
     __snake_case						:							List[Any]       			=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case						:							List[Any]       			=MPNetTokenizer
     __snake_case						:							Optional[Any]       			=['''input_ids''', '''attention_mask''']





     def __init__(						self			:					Any       ,       lowerCAmelCase__			:					Any=None       ,       lowerCAmelCase__			:					Tuple=None       ,       lowerCAmelCase__			:					Union[str, Any]=True       ,       lowerCAmelCase__			:					Dict="<s>"       ,       lowerCAmelCase__			:					str="</s>"       ,       lowerCAmelCase__			:					Optional[int]="</s>"       ,       lowerCAmelCase__			:					Dict="<s>"       ,       lowerCAmelCase__			:					Optional[Any]="[UNK]"       ,       lowerCAmelCase__			:					List[str]="<pad>"       ,       lowerCAmelCase__			:					Union[str, Any]="<mask>"       ,       lowerCAmelCase__			:					Optional[int]=True       ,       lowerCAmelCase__			:					Optional[int]=None       ,       **lowerCAmelCase__			:					Union[str, Any]       ,       )    ->					str:

      """simple docstring"""
      super().__init__(
          __lowerCAmelCase       ,       tokenizer_file=__lowerCAmelCase       ,       do_lower_case=__lowerCAmelCase       ,       bos_token=__lowerCAmelCase       ,       eos_token=__lowerCAmelCase       ,       sep_token=__lowerCAmelCase       ,       cls_token=__lowerCAmelCase       ,       unk_token=__lowerCAmelCase       ,       pad_token=__lowerCAmelCase       ,       mask_token=__lowerCAmelCase       ,       tokenize_chinese_chars=__lowerCAmelCase       ,       strip_accents=__lowerCAmelCase       ,       **__lowerCAmelCase       ,       )

      SCREAMING_SNAKE_CASE							  =						json.loads(self.backend_tokenizer.normalizer.__getstate__())
      if (
          pre_tok_state.get('lowercase'       ,       __lowerCAmelCase) != do_lower_case
          or pre_tok_state.get('strip_accents'       ,       __lowerCAmelCase) != strip_accents
      ):
       SCREAMING_SNAKE_CASE							  =						getattr(__lowerCAmelCase       ,       pre_tok_state.pop('type'))
       SCREAMING_SNAKE_CASE							  =						do_lower_case
       SCREAMING_SNAKE_CASE							  =						strip_accents
       SCREAMING_SNAKE_CASE							  =						pre_tok_class(**__lowerCAmelCase)

      SCREAMING_SNAKE_CASE							  =						do_lower_case





     @property
     def   __UpperCamelCase		(						self			:					Union[str, Any])    ->					Dict:

      """simple docstring"""


      if self._mask_token is None:
       if self.verbose:
        logger.error('Using mask_token, but it is not set yet.')
       return None
      return str(self._mask_token)





     @mask_token.setter
     def   __UpperCamelCase		(						self			:					Tuple       ,       lowerCAmelCase__			:					List[str])    ->					str:

      """simple docstring"""
      SCREAMING_SNAKE_CASE							  =						AddedToken(__lowerCAmelCase       ,       lstrip=__lowerCAmelCase       ,       rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,       __lowerCAmelCase) else value
      SCREAMING_SNAKE_CASE							  =						value





     def   __UpperCamelCase		(						self			:					Union[str, Any]       ,       lowerCAmelCase__			:					Optional[Any]       ,       lowerCAmelCase__			:					List[Any]=None)    ->					List[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE							  =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
      if token_ids_a is None:
       return output

      return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





     def   __UpperCamelCase		(						self			:					Any       ,       lowerCAmelCase__			:					List[str]       ,       lowerCAmelCase__			:					str = None)    ->					Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE							  =						[self.sep_token_id]
      SCREAMING_SNAKE_CASE							  =						[self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





     def   __UpperCamelCase		(						self			:					Optional[int]       ,       lowerCAmelCase__			:					Union[str, Any]       ,       lowerCAmelCase__			:					Union[str, Any] = None)    ->					Union[str, Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE							  =						self._tokenizer.model.save(__lowerCAmelCase       ,       name=__lowerCAmelCase)
      return tuple(__lowerCAmelCase)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCAmelCase      				=							{"configuration_visual_bert": ["VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "VisualBertConfig"]}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      				=							[
          "VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
          "VisualBertForMultipleChoice",
          "VisualBertForPreTraining",
          "VisualBertForQuestionAnswering",
          "VisualBertForRegionToPhraseAlignment",
          "VisualBertForVisualReasoning",
          "VisualBertLayer",
          "VisualBertModel",
          "VisualBertPreTrainedModel",
      ]


if TYPE_CHECKING:
      from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_visual_bert import (
                VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                VisualBertForMultipleChoice,
                VisualBertForPreTraining,
                VisualBertForQuestionAnswering,
                VisualBertForRegionToPhraseAlignment,
                VisualBertForVisualReasoning,
                VisualBertLayer,
                VisualBertModel,
                VisualBertPreTrainedModel,
            )


else:
      import sys

      __UpperCAmelCase      				=							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




