

import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
    import tensorflow as tf

    tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
    import torch

if is_datasets_available():
    from datasets import load_dataset


lowerCAmelCase       :				Optional[Any]         =					5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors

def 				A_					(							a					):

      """simple docstring"""



      return PTtoTFCommand(
          args.model_name   ,				args.local_dir   ,				args.max_error   ,				args.new_weights   ,				args.no_pr   ,				args.push   ,				args.extra_commit_description   ,				args.override_model_class   ,				)



class 	_A						(					__magic_name__):

  @staticmethod
  def 	UpperCAmelCase      (			_SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       parser.add_parser(
            'pt-to-tf'	,   help=(
                'CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint.'
                ' Can also be used to validate existing weights without opening PRs, with --no-pr.'
            )	,   )
        train_parser.add_argument(
            '--model-name'	,   type=_SCREAMING_SNAKE_CASE	,   required=_SCREAMING_SNAKE_CASE	,   help='The model name, including owner/organization, as seen on the hub.'	,   )
        train_parser.add_argument(
            '--local-dir'	,   type=_SCREAMING_SNAKE_CASE	,   default=''	,   help='Optional local directory of the model repository. Defaults to /tmp/{model_name}'	,   )
        train_parser.add_argument(
            '--max-error'	,   type=_SCREAMING_SNAKE_CASE	,   default=_SCREAMING_SNAKE_CASE	,   help=(
                f"Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."
            )	,   )
        train_parser.add_argument(
            '--new-weights'	,   action='store_true'	,   help='Optional flag to create new TensorFlow weights, even if they already exist.'	,   )
        train_parser.add_argument(
            '--no-pr'	,   action='store_true'	,   help='Optional flag to NOT open a PR with converted weights.'   )
        train_parser.add_argument(
            '--push'	,   action='store_true'	,   help='Optional flag to push the weights directly to `main` (requires permissions)'	,   )
        train_parser.add_argument(
            '--extra-commit-description'	,   type=_SCREAMING_SNAKE_CASE	,   default=''	,   help='Optional additional commit description to use when opening a PR (e.g. to tag the owner).'	,   )
        train_parser.add_argument(
            '--override-model-class'	,   type=_SCREAMING_SNAKE_CASE	,   default=_SCREAMING_SNAKE_CASE	,   help='If you think you know better than the auto-detector, you can specify the model class here. '
            'Can be either an AutoModel class or a specific model class like BertForSequenceClassification.'	,   )
        train_parser.set_defaults(func=_SCREAMING_SNAKE_CASE   )

  @staticmethod
  def 	UpperCAmelCase      (			_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Any          =       set(pt_outputs.keys()   )
        SCREAMING_SNAKE_CASE_			:					Any          =       set(tf_outputs.keys()   )
        if pt_out_attrs != tf_out_attrs:
              raise ValueError(
                  f"The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:"
                  f" {tf_out_attrs})"   )

        # 2. For each output attribute, computes the difference
        def _find_pt_tf_differences(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE=""   ):
              # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
              # recursivelly, keeping the name of the attribute.
              if isinstance(_SCREAMING_SNAKE_CASE	,   torch.Tensor   ):
                    SCREAMING_SNAKE_CASE_			:					int          =       np.max(np.abs(pt_out.numpy() - tf_out.numpy()   )   )
                    SCREAMING_SNAKE_CASE_			:					Any          =       tensor_difference
              else:
                    SCREAMING_SNAKE_CASE_			:					Dict          =       attr_name
                    for i, pt_item in enumerate(_SCREAMING_SNAKE_CASE   ):
                          # If it is a named attribute, we keep the name. Otherwise, just its index.
                          if isinstance(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ):
                                SCREAMING_SNAKE_CASE_			:					List[str]          =       root_name + pt_item
                                SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       tf_out[pt_item]
                                SCREAMING_SNAKE_CASE_			:					Optional[int]          =       pt_out[pt_item]
                          else:
                                SCREAMING_SNAKE_CASE_			:					int          =       root_name + f"[{i}]"
                                SCREAMING_SNAKE_CASE_			:					List[Any]          =       tf_out[i]
                          SCREAMING_SNAKE_CASE_			:					List[str]          =       _find_pt_tf_differences(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )

              return differences

        return _find_pt_tf_differences(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   {}   )

  def __init__(			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   *_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					str          =       logging.get_logger('transformers-cli/pt_to_tf'   )
        SCREAMING_SNAKE_CASE_			:					Dict          =       model_name
        SCREAMING_SNAKE_CASE_			:					int          =       local_dir if local_dir else os.path.join('/tmp'	,   _SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       max_error
        SCREAMING_SNAKE_CASE_			:					str          =       new_weights
        SCREAMING_SNAKE_CASE_			:					Tuple          =       no_pr
        SCREAMING_SNAKE_CASE_			:					str          =       push
        SCREAMING_SNAKE_CASE_			:					Any          =       extra_commit_description
        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       override_model_class

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        def _get_audio_input():
              SCREAMING_SNAKE_CASE_			:					Optional[int]          =       load_dataset('hf-internal-testing/librispeech_asr_dummy'	,   'clean'	,   split='validation'   )
              SCREAMING_SNAKE_CASE_			:					Tuple          =       ds.sort('id'   ).select(range(2   )   )[:2]['audio']
              SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       [x['array'] for x in speech_samples]
              return raw_samples

        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       type(pt_model.config   )
        if model_config_class in PROCESSOR_MAPPING:
              SCREAMING_SNAKE_CASE_			:					str          =       AutoProcessor.from_pretrained(self._local_dir   )
              if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
                    SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       processor.tokenizer.eos_token
        elif model_config_class in IMAGE_PROCESSOR_MAPPING:
              SCREAMING_SNAKE_CASE_			:					List[str]          =       AutoImageProcessor.from_pretrained(self._local_dir   )
        elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
              SCREAMING_SNAKE_CASE_			:					Dict          =       AutoFeatureExtractor.from_pretrained(self._local_dir   )
        elif model_config_class in TOKENIZER_MAPPING:
              SCREAMING_SNAKE_CASE_			:					List[Any]          =       AutoTokenizer.from_pretrained(self._local_dir   )
              if processor.pad_token is None:
                    SCREAMING_SNAKE_CASE_			:					int          =       processor.eos_token
        else:
              raise ValueError(f"Unknown data processing type (model config type: {model_config_class})"   )

        SCREAMING_SNAKE_CASE_			:					List[str]          =       set(inspect.signature(pt_model.forward   ).parameters.keys()   )
        SCREAMING_SNAKE_CASE_			:					Tuple          =       {}
        if "input_ids" in model_forward_signature:
              processor_inputs.update(
                  {
                      'text': ['Hi there!', 'I am a batch with more than one row and different input lengths.'],
                      'padding': True,
                      'truncation': True,
                  }   )
        if "pixel_values" in model_forward_signature:
              SCREAMING_SNAKE_CASE_			:					Optional[int]          =       load_dataset('cifar10'	,   'plain_text'	,   split='test'   )[:2]['img']
              processor_inputs.update({'images': sample_images}   )
        if "input_features" in model_forward_signature:
              SCREAMING_SNAKE_CASE_			:					str          =       inspect.signature(processor.feature_extractor   ).parameters
              # Pad to the largest input length by default but take feature extractor default
              # padding value if it exists e.g. "max_length" and is not False or None
              if "padding" in feature_extractor_signature:
                    SCREAMING_SNAKE_CASE_			:					str          =       feature_extractor_signature['padding'].default
                    if default_strategy is not False and default_strategy is not None:
                          SCREAMING_SNAKE_CASE_			:					Tuple          =       default_strategy
                    else:
                          SCREAMING_SNAKE_CASE_			:					Optional[int]          =       True
              else:
                    SCREAMING_SNAKE_CASE_			:					List[str]          =       True
              processor_inputs.update({'audio': _get_audio_input(), 'padding': padding_strategy}   )
        if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
              processor_inputs.update({'audio': _get_audio_input(), 'padding': True}   )
        SCREAMING_SNAKE_CASE_			:					List[str]          =       processor(**_SCREAMING_SNAKE_CASE	,   return_tensors='pt'   )
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       processor(**_SCREAMING_SNAKE_CASE	,   return_tensors='tf'   )

        # Extra input requirements, in addition to the input modality
        if (
            config.is_encoder_decoder
            or (hasattr(_SCREAMING_SNAKE_CASE	,   'encoder'   ) and hasattr(_SCREAMING_SNAKE_CASE	,   'decoder'   ))
            or "decoder_input_ids" in tf_dummy_inputs
        ):
              SCREAMING_SNAKE_CASE_			:					Tuple          =       np.asarray([[1], [1]]	,   dtype=_SCREAMING_SNAKE_CASE   ) * (pt_model.config.decoder_start_token_id or 0)
              pt_input.update({'decoder_input_ids': torch.tensor(_SCREAMING_SNAKE_CASE   )}   )
              tf_input.update({'decoder_input_ids': tf.convert_to_tensor(_SCREAMING_SNAKE_CASE   )}   )

        return pt_input, tf_input



  def 	UpperCAmelCase      (			self   ):





        """simple docstring"""







        if version.parse(huggingface_hub.__version__   ) < version.parse('0.9.0'   ):
              raise ImportError(
                  'The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub'
                  ' installation.'   )
        else:
              from huggingface_hub import Repository, create_commit
              from huggingface_hub._commit_api import CommitOperationAdd

        # Fetch remote data
        SCREAMING_SNAKE_CASE_			:					Tuple          =       Repository(local_dir=self._local_dir	,   clone_from=self._model_name   )

        # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       AutoConfig.from_pretrained(self._local_dir   )
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       config.architectures
        if self._override_model_class is not None:
              if self._override_model_class.startswith('TF'   ):
                    SCREAMING_SNAKE_CASE_			:					int          =       [self._override_model_class[2:]]
              else:
                    SCREAMING_SNAKE_CASE_			:					Any          =       [self._override_model_class]
              try:
                    SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       getattr(import_module('transformers'   )	,   architectures[0]   )
              except AttributeError:
                    raise ValueError(f"Model class {self._override_model_class} not found in transformers."   )
              try:
                    SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       getattr(import_module('transformers'   )	,   'TF' + architectures[0]   )
              except AttributeError:
                    raise ValueError(f"TF model class TF{self._override_model_class} not found in transformers."   )
        elif architectures is None:  # No architecture defined -- use auto classes
              SCREAMING_SNAKE_CASE_			:					Tuple          =       getattr(import_module('transformers'   )	,   'AutoModel'   )
              SCREAMING_SNAKE_CASE_			:					int          =       getattr(import_module('transformers'   )	,   'TFAutoModel'   )
              self._logger.warning('No detected architecture, using AutoModel/TFAutoModel'   )
        else:  # Architecture defined -- use it
              if len(_SCREAMING_SNAKE_CASE   ) > 1:
                    raise ValueError(f"More than one architecture was found, aborting. (architectures = {architectures})"   )
              self._logger.warning(f"Detected architecture: {architectures[0]}"   )
              SCREAMING_SNAKE_CASE_			:					List[str]          =       getattr(import_module('transformers'   )	,   architectures[0]   )
              try:
                    SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       getattr(import_module('transformers'   )	,   'TF' + architectures[0]   )
              except AttributeError:
                    raise AttributeError(f"The TensorFlow equivalent of {architectures[0]} doesn't exist in transformers."   )

        # Check the TF dummy inputs to see what keys we need in the forward pass
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       tf_class.from_config(_SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Tuple          =       tf_from_pt_model.dummy_inputs

        del tf_from_pt_model  # Try to keep only one model in memory at a time

        # Load the model and get some basic inputs
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       pt_class.from_pretrained(self._local_dir   )
        pt_model.eval()

        SCREAMING_SNAKE_CASE_    ,  SCREAMING_SNAKE_CASE_			:					List[str]          =       self.get_inputs(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )

        with torch.no_grad():
              SCREAMING_SNAKE_CASE_			:					Tuple          =       pt_model(**_SCREAMING_SNAKE_CASE	,   output_hidden_states=_SCREAMING_SNAKE_CASE   )
        del pt_model  # will no longer be used, and may have a large memory footprint

        SCREAMING_SNAKE_CASE_			:					int          =       tf_class.from_pretrained(self._local_dir	,   from_pt=_SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Tuple          =       tf_from_pt_model(**_SCREAMING_SNAKE_CASE	,   output_hidden_states=_SCREAMING_SNAKE_CASE	,   training=_SCREAMING_SNAKE_CASE   )

        # Confirms that cross loading PT weights into TF worked.
        SCREAMING_SNAKE_CASE_			:					Dict          =       self.find_pt_tf_differences(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       {k: v for k, v in crossload_differences.items() if 'hidden' not in k}
        SCREAMING_SNAKE_CASE_			:					int          =       {k: v for k, v in crossload_differences.items() if 'hidden' in k}
        if len(_SCREAMING_SNAKE_CASE   ) == 0 and architectures is not None:
              raise ValueError(
                  f"Something went wrong -- the config file has architectures ({architectures}), but no model head"
                  ' output was found. All outputs start with \'hidden\''   )
        SCREAMING_SNAKE_CASE_			:					Dict          =       max(output_differences.values()   ) if output_differences else 0.0
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       max(hidden_differences.values()   )
        if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
              raise ValueError(
                  'The cross-loaded TensorFlow model has different outputs, something went wrong!\n'
                  + f"\nList of maximum output differences above the threshold ({self._max_error}):\n"
                  + '\n'.join([f"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]   )
                  + f"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
                  + '\n'.join([f"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]   )   )

        # Save the weights in a TF format (if needed) and confirms that the results are still good
        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       os.path.join(self._local_dir	,   _SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					int          =       os.path.join(self._local_dir	,   _SCREAMING_SNAKE_CASE   )
        if (not os.path.exists(_SCREAMING_SNAKE_CASE   ) and not os.path.exists(_SCREAMING_SNAKE_CASE   )) or self._new_weights:
              tf_from_pt_model.save_pretrained(self._local_dir   )
        del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

        SCREAMING_SNAKE_CASE_			:					Tuple          =       tf_class.from_pretrained(self._local_dir   )
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       tf_model(**_SCREAMING_SNAKE_CASE	,   output_hidden_states=_SCREAMING_SNAKE_CASE   )

        SCREAMING_SNAKE_CASE_			:					Dict          =       self.find_pt_tf_differences(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Tuple          =       {k: v for k, v in conversion_differences.items() if 'hidden' not in k}
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       {k: v for k, v in conversion_differences.items() if 'hidden' in k}
        if len(_SCREAMING_SNAKE_CASE   ) == 0 and architectures is not None:
              raise ValueError(
                  f"Something went wrong -- the config file has architectures ({architectures}), but no model head"
                  ' output was found. All outputs start with \'hidden\''   )
        SCREAMING_SNAKE_CASE_			:					Any          =       max(output_differences.values()   ) if output_differences else 0.0
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       max(hidden_differences.values()   )
        if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
              raise ValueError(
                  'The converted TensorFlow model has different outputs, something went wrong!\n'
                  + f"\nList of maximum output differences above the threshold ({self._max_error}):\n"
                  + '\n'.join([f"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]   )
                  + f"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
                  + '\n'.join([f"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]   )   )

        SCREAMING_SNAKE_CASE_			:					List[str]          =       'Update TF weights' if self._new_weights else 'Add TF weights'
        if self._push:
              repo.git_add(auto_lfs_track=_SCREAMING_SNAKE_CASE   )
              repo.git_commit(_SCREAMING_SNAKE_CASE   )
              repo.git_push(blocking=_SCREAMING_SNAKE_CASE   )  # this prints a progress bar with the upload
              self._logger.warning(f"TF weights pushed into {self._model_name}"   )
        elif not self._no_pr:
              self._logger.warning('Uploading the weights into a new PR...'   )
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       (
                  'Model converted by the [`transformers`\' `pt_to_tf`'
                  ' CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). '
                  'All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n'
                  f"Maximum crossload output difference={max_crossload_output_diff:.3e}; "
                  f"Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n"
                  f"Maximum conversion output difference={max_conversion_output_diff:.3e}; "
                  f"Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n"
              )
              if self._max_error > MAX_ERROR:
                    commit_descrition += (
                        f"\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!"
                    )
              if self._extra_commit_description:
                    commit_descrition += "\n\n" + self._extra_commit_description

              # sharded model -> adds all related files (index and .h5 shards)
              if os.path.exists(_SCREAMING_SNAKE_CASE   ):
                    SCREAMING_SNAKE_CASE_			:					List[Any]          =       [
                        CommitOperationAdd(path_in_repo=_SCREAMING_SNAKE_CASE	,   path_or_fileobj=_SCREAMING_SNAKE_CASE   )
                    ]
                    for shard_path in tf.io.gfile.glob(self._local_dir + '/tf_model-*.h5'   ):
                          operations += [
                              CommitOperationAdd(path_in_repo=os.path.basename(_SCREAMING_SNAKE_CASE   )	,   path_or_fileobj=_SCREAMING_SNAKE_CASE   )
                          ]
              else:
                    SCREAMING_SNAKE_CASE_			:					int          =       [CommitOperationAdd(path_in_repo=_SCREAMING_SNAKE_CASE	,   path_or_fileobj=_SCREAMING_SNAKE_CASE   )]

              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       create_commit(
                  repo_id=self._model_name	,   operations=_SCREAMING_SNAKE_CASE	,   commit_message=_SCREAMING_SNAKE_CASE	,   commit_description=_SCREAMING_SNAKE_CASE	,   repo_type='model'	,   create_pr=_SCREAMING_SNAKE_CASE	,   ).pr_url
              self._logger.warning(f"PR open in {hub_pr_url}"   )









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


lowerCAmelCase       :				List[Any]         =					logging.get_logger(__name__)

lowerCAmelCase       :				Any         =					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase       :				Any         =					{
    'vocab_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase       :				str         =					{
    'microsoft/mpnet-base': 5_12,
}

lowerCAmelCase       :				Optional[Any]         =					{
    'microsoft/mpnet-base': {'do_lower_case': True},
}
class 	_A						(					__magic_name__):
  SCREAMING_SNAKE_CASE      : Tuple    =					VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE      : List[Any]    =					PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE      : Dict    =					PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE      : str    =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE      : Dict    =					MPNetTokenizer
  SCREAMING_SNAKE_CASE      : Any    =					['''input_ids''', '''attention_mask''']

  def __init__(			self	,   _SCREAMING_SNAKE_CASE=None	,   _SCREAMING_SNAKE_CASE=None	,   _SCREAMING_SNAKE_CASE=True	,   _SCREAMING_SNAKE_CASE="<s>"	,   _SCREAMING_SNAKE_CASE="</s>"	,   _SCREAMING_SNAKE_CASE="</s>"	,   _SCREAMING_SNAKE_CASE="<s>"	,   _SCREAMING_SNAKE_CASE="[UNK]"	,   _SCREAMING_SNAKE_CASE="<pad>"	,   _SCREAMING_SNAKE_CASE="<mask>"	,   _SCREAMING_SNAKE_CASE=True	,   _SCREAMING_SNAKE_CASE=None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        super().__init__(
            _SCREAMING_SNAKE_CASE	,   tokenizer_file=_SCREAMING_SNAKE_CASE	,   do_lower_case=_SCREAMING_SNAKE_CASE	,   bos_token=_SCREAMING_SNAKE_CASE	,   eos_token=_SCREAMING_SNAKE_CASE	,   sep_token=_SCREAMING_SNAKE_CASE	,   cls_token=_SCREAMING_SNAKE_CASE	,   unk_token=_SCREAMING_SNAKE_CASE	,   pad_token=_SCREAMING_SNAKE_CASE	,   mask_token=_SCREAMING_SNAKE_CASE	,   tokenize_chinese_chars=_SCREAMING_SNAKE_CASE	,   strip_accents=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE	,   )

        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
        if (
            pre_tok_state.get('lowercase'	,   _SCREAMING_SNAKE_CASE   ) != do_lower_case
            or pre_tok_state.get('strip_accents'	,   _SCREAMING_SNAKE_CASE   ) != strip_accents
        ):
              SCREAMING_SNAKE_CASE_			:					Any          =       getattr(_SCREAMING_SNAKE_CASE	,   pre_tok_state.pop('type'   )   )
              SCREAMING_SNAKE_CASE_			:					Dict          =       do_lower_case
              SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       strip_accents
              SCREAMING_SNAKE_CASE_			:					Tuple          =       pre_tok_class(**_SCREAMING_SNAKE_CASE   )

        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       do_lower_case

  @property
  def 	UpperCAmelCase      (			self   ):





        """simple docstring"""







        if self._mask_token is None:
              if self.verbose:
                    logger.error('Using mask_token, but it is not set yet.'   )
              return None
        return str(self._mask_token   )

  @mask_token.setter
  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       AddedToken(_SCREAMING_SNAKE_CASE	,   lstrip=_SCREAMING_SNAKE_CASE	,   rstrip=_SCREAMING_SNAKE_CASE   ) if isinstance(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ) else value
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       value

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE=None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					List[str]          =       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
              return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					str          =       [self.sep_token_id]
        SCREAMING_SNAKE_CASE_			:					int          =       [self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Tuple          =       self._tokenizer.model.save(_SCREAMING_SNAKE_CASE	,   name=_SCREAMING_SNAKE_CASE   )
        return tuple(_SCREAMING_SNAKE_CASE   )






