

import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

UpperCAmelCase__	:       Optional[Any]			      =      logging.get_logger(__name__)

UpperCAmelCase__	:       Tuple			      =      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCAmelCase__	:       List[Any]			      =      {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

UpperCAmelCase__	:       Tuple			      =      {
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

UpperCAmelCase__	:       Union[str, Any]			      =      {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}






def    __lowercase (       )       ->  str:
      SCREAMING_SNAKE_CASE	: Tuple					       =				(
          list(range(ord("""!"""     )  ,      ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )  ,      ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )  ,      ord("""ÿ"""     ) + 1     )     )
      )
      SCREAMING_SNAKE_CASE	: List[str]					       =				bs[:]
      SCREAMING_SNAKE_CASE	: Optional[int]					       =				0
      for b in range(2**8     ):
            if b not in bs:
                  bs.append(_A     )
                  cs.append(2**8 + n     )
                  n += 1
      SCREAMING_SNAKE_CASE	: List[Any]					       =				[chr(_A     ) for n in cs]
      return dict(zip(_A  ,      _A     )     )






def    __lowercase (       _A     )       ->  Optional[int]:
      SCREAMING_SNAKE_CASE	: Dict					       =				set()
      SCREAMING_SNAKE_CASE	: Dict					       =				word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)     )
            SCREAMING_SNAKE_CASE	: List[str]					       =				char
      return pairs







class     a__    (				UpperCAmelCase    ):







  """simple docstring"""




  UpperCAmelCase__ :	str         =VOCAB_FILES_NAMES
  UpperCAmelCase__ :	List[str]         =PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__ :	List[Any]         =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__ :	Optional[Any]         =["""input_ids""", """attention_mask""", """token_type_ids"""]


  def __init__(     self      :	Tuple			,			UpperCAmelCase__      :	Dict			,			UpperCAmelCase__      :	Any			,			UpperCAmelCase__      :	Tuple="replace"			,			UpperCAmelCase__      :	Union[str, Any]="[CLS]"			,			UpperCAmelCase__      :	List[str]="[SEP]"			,			UpperCAmelCase__      :	str="[SEP]"			,			UpperCAmelCase__      :	str="[CLS]"			,			UpperCAmelCase__      :	int="[UNK]"			,			UpperCAmelCase__      :	List[Any]="[PAD]"			,			UpperCAmelCase__      :	int="[MASK]"			,			UpperCAmelCase__      :	Any=False			,			UpperCAmelCase__      :	Optional[int]=False			,			**UpperCAmelCase__      :	str			,			)   ->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: List[str]					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else bos_token
        SCREAMING_SNAKE_CASE	: Any					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else eos_token
        SCREAMING_SNAKE_CASE	: int					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else sep_token
        SCREAMING_SNAKE_CASE	: str					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else cls_token
        SCREAMING_SNAKE_CASE	: List[str]					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else unk_token
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				AddedToken(UpperCAmelCase__			,			lstrip=UpperCAmelCase__			,			rstrip=UpperCAmelCase__						) if isinstance(UpperCAmelCase__			,			UpperCAmelCase__						) else mask_token

        super().__init__(
            errors=UpperCAmelCase__			,			bos_token=UpperCAmelCase__			,			eos_token=UpperCAmelCase__			,			unk_token=UpperCAmelCase__			,			sep_token=UpperCAmelCase__			,			cls_token=UpperCAmelCase__			,			pad_token=UpperCAmelCase__			,			mask_token=UpperCAmelCase__			,			add_prefix_space=UpperCAmelCase__			,			add_bos_token=UpperCAmelCase__			,			**UpperCAmelCase__			,			)
        SCREAMING_SNAKE_CASE	: Dict					       =				add_bos_token

        with open(UpperCAmelCase__			,			encoding="""utf-8"""						) as vocab_handle:
              SCREAMING_SNAKE_CASE	: Optional[Any]					       =				json.load(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				{v: k for k, v in self.encoder.items()}
        SCREAMING_SNAKE_CASE	: Any					       =				errors  # how to handle errors in decoding
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				bytes_to_unicode()
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				{v: k for k, v in self.byte_encoder.items()}
        with open(UpperCAmelCase__			,			encoding="""utf-8"""						) as merges_handle:
              SCREAMING_SNAKE_CASE	: List[Any]					       =				merges_handle.read().split("""\n"""						)[1:-1]
        SCREAMING_SNAKE_CASE	: str					       =				[tuple(merge.split()						) for merge in bpe_merges]
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				dict(zip(UpperCAmelCase__			,			range(len(UpperCAmelCase__						)						)						)						)
        SCREAMING_SNAKE_CASE	: Tuple					       =				{}
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        SCREAMING_SNAKE_CASE	: Dict					       =				re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)


  @property
  # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
  def 	_lowercase (     self      :	Any						)   ->Optional[int]:
        """simple docstring"""







        return len(self.encoder						)


  def 	_lowercase (     self      :	Dict						)   ->Optional[int]:
        """simple docstring"""







        return dict(self.encoder			,			**self.added_tokens_encoder						)


  def 	_lowercase (     self      :	int			,			UpperCAmelCase__      :	Union[str, Any]						)   ->Optional[Any]:
        """simple docstring"""







        if token in self.cache:
              return self.cache[token]
        SCREAMING_SNAKE_CASE	: List[Any]					       =				tuple(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: int					       =				get_pairs(UpperCAmelCase__						)

        if not pairs:
              return token

        while True:
              SCREAMING_SNAKE_CASE	: Dict					       =				min(UpperCAmelCase__			,			key=lambda UpperCAmelCase__						: self.bpe_ranks.get(UpperCAmelCase__			,			float("""inf"""						)						)						)
              if bigram not in self.bpe_ranks:
                    break
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Optional[int]					       =				bigram
              SCREAMING_SNAKE_CASE	: str					       =				[]
              SCREAMING_SNAKE_CASE	: Dict					       =				0
              while i < len(UpperCAmelCase__						):
                    try:
                          SCREAMING_SNAKE_CASE	: List[Any]					       =				word.index(UpperCAmelCase__			,			UpperCAmelCase__						)
                    except ValueError:
                          new_word.extend(word[i:]						)
                          break
                    else:
                          new_word.extend(word[i:j]						)
                          SCREAMING_SNAKE_CASE	: Any					       =				j

                    if word[i] == first and i < len(UpperCAmelCase__						) - 1 and word[i + 1] == second:
                          new_word.append(first + second						)
                          i += 2
                    else:
                          new_word.append(word[i]						)
                          i += 1
              SCREAMING_SNAKE_CASE	: int					       =				tuple(UpperCAmelCase__						)
              SCREAMING_SNAKE_CASE	: Tuple					       =				new_word
              if len(UpperCAmelCase__						) == 1:
                    break
              else:
                    SCREAMING_SNAKE_CASE	: int					       =				get_pairs(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Any					       =				""" """.join(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Tuple					       =				word
        return word


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	List[int]			,			UpperCAmelCase__      :	Optional[List[int]] = None						)   ->List[int]:
        """simple docstring"""







        if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        SCREAMING_SNAKE_CASE	: List[str]					       =				[self.cls_token_id]
        SCREAMING_SNAKE_CASE	: Dict					       =				[self.sep_token_id]
        return cls + token_ids_a + sep + token_ids_a + sep


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	List[int]			,			UpperCAmelCase__      :	Optional[List[int]] = None			,			UpperCAmelCase__      :	bool = False						)   ->List[int]:
        """simple docstring"""







        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=UpperCAmelCase__			,			token_ids_a=UpperCAmelCase__			,			already_has_special_tokens=UpperCAmelCase__						)

        if token_ids_a is None:
              return [1] + ([0] * len(UpperCAmelCase__						)) + [1]
        return [1] + ([0] * len(UpperCAmelCase__						)) + [1] + ([0] * len(UpperCAmelCase__						)) + [1]


  def 	_lowercase (     self      :	int			,			UpperCAmelCase__      :	List[int]			,			UpperCAmelCase__      :	Optional[List[int]] = None						)   ->List[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Dict					       =				[self.sep_token_id]
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				[self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep						) * [0]
        return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	Union[str, Any]						)   ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Optional[int]					       =				[]
        for token in re.findall(self.pat			,			UpperCAmelCase__						):
              SCREAMING_SNAKE_CASE	: int					       =				"""""".join(
                  self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase__						).split(""" """						)						)
        return bpe_tokens


  def 	_lowercase (     self      :	List[str]			,			UpperCAmelCase__      :	str						)   ->Tuple:
        """simple docstring"""







        return self.encoder.get(UpperCAmelCase__			,			self.encoder.get(self.unk_token						)						)


  def 	_lowercase (     self      :	int			,			UpperCAmelCase__      :	Any						)   ->Any:
        """simple docstring"""







        return self.decoder.get(UpperCAmelCase__						)


  def 	_lowercase (     self      :	Optional[int]			,			UpperCAmelCase__      :	str						)   ->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: str					       =				"""""".join(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: int					       =				bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""			,			errors=self.errors						)
        return text


  def 	_lowercase (     self      :	Optional[int]			,			UpperCAmelCase__      :	str			,			UpperCAmelCase__      :	Optional[str] = None						)   ->Tuple[str]:
        """simple docstring"""







        if not os.path.isdir(UpperCAmelCase__						):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
              return
        SCREAMING_SNAKE_CASE	: List[Any]					       =				os.path.join(
            UpperCAmelCase__			,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				os.path.join(
            UpperCAmelCase__			,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

        with open(UpperCAmelCase__			,			"""w"""			,			encoding="""utf-8"""						) as f:
              f.write(json.dumps(self.encoder			,			indent=2			,			sort_keys=UpperCAmelCase__			,			ensure_ascii=UpperCAmelCase__						) + """\n"""						)

        SCREAMING_SNAKE_CASE	: List[str]					       =				0
        with open(UpperCAmelCase__			,			"""w"""			,			encoding="""utf-8"""						) as writer:
              writer.write("""#version: 0.2\n"""						)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,			key=lambda UpperCAmelCase__						: kv[1]						):
                    if index != token_index:
                          logger.warning(
                              f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                              """ Please check that the tokenizer is not corrupted!"""						)
                          SCREAMING_SNAKE_CASE	: Optional[Any]					       =				token_index
                    writer.write(""" """.join(UpperCAmelCase__						) + """\n"""						)
                    index += 1

        return vocab_file, merge_file


  def 	_lowercase (     self      :	str			,			UpperCAmelCase__      :	List[Any]			,			UpperCAmelCase__      :	Tuple=False			,			**UpperCAmelCase__      :	Any						)   ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				kwargs.pop("""add_prefix_space"""			,			self.add_prefix_space						)
        if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase__						) > 0 and not text[0].isspace()):
              SCREAMING_SNAKE_CASE	: Optional[Any]					       =				""" """ + text
        return (text, kwargs)




  def 	_lowercase (     self      :	List[str]			,			UpperCAmelCase__      :	"Conversation"						)   ->List[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Dict					       =				[]
        for is_user, text in conversation.iter_texts():
              input_ids.extend(self.encode(UpperCAmelCase__			,			add_special_tokens=UpperCAmelCase__						) + [self.eos_token_id]						)
        if len(UpperCAmelCase__						) > self.model_max_length:
              SCREAMING_SNAKE_CASE	: List[Any]					       =				input_ids[-self.model_max_length :]
        return input_ids







import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer







class     a__    (				tf.keras.layers.Layer    ):







  """simple docstring"""




  def __init__(     self      :	str			,			UpperCAmelCase__      :	List			,			UpperCAmelCase__      :	bool			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	str = "longest"			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	int = 5_1_2			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	bool = True			,			**UpperCAmelCase__      :	Optional[int]			,			)   ->Union[str, Any]:
        """simple docstring"""







        super().__init__()
        if use_fast_bert_tokenizer:
              SCREAMING_SNAKE_CASE	: Any					       =				FastBertTokenizer(
                  UpperCAmelCase__			,			token_out_type=tf.intaa			,			lower_case_nfd_strip_accents=UpperCAmelCase__			,			**UpperCAmelCase__						)
        else:
              SCREAMING_SNAKE_CASE	: Any					       =				tf.lookup.StaticVocabularyTable(
                  tf.lookup.KeyValueTensorInitializer(
                      keys=UpperCAmelCase__			,			key_dtype=tf.string			,			values=tf.range(tf.size(UpperCAmelCase__			,			out_type=tf.intaa						)			,			dtype=tf.intaa						)			,			value_dtype=tf.intaa			,			)			,			num_oov_buckets=1			,			)
              SCREAMING_SNAKE_CASE	: Dict					       =				BertTokenizerLayer(
                  UpperCAmelCase__			,			token_out_type=tf.intaa			,			lower_case=UpperCAmelCase__			,			**UpperCAmelCase__						)

        SCREAMING_SNAKE_CASE	: Any					       =				vocab_list
        SCREAMING_SNAKE_CASE	: int					       =				do_lower_case
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				cls_token_id or vocab_list.index("""[CLS]"""						)
        SCREAMING_SNAKE_CASE	: int					       =				sep_token_id or vocab_list.index("""[SEP]"""						)
        SCREAMING_SNAKE_CASE	: int					       =				pad_token_id or vocab_list.index("""[PAD]"""						)
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				ShrinkLongestTrimmer(max_length - 3			,			axis=1						)  # Allow room for special tokens
        SCREAMING_SNAKE_CASE	: Tuple					       =				max_length
        SCREAMING_SNAKE_CASE	: List[str]					       =				padding
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				truncation
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				pad_to_multiple_of
        SCREAMING_SNAKE_CASE	: Dict					       =				return_token_type_ids
        SCREAMING_SNAKE_CASE	: str					       =				return_attention_mask


  @classmethod
  def 	_lowercase (     cls      :	Optional[int]			,			UpperCAmelCase__      :	"PreTrainedTokenizerBase"			,			**UpperCAmelCase__      :	Tuple						)   ->str:  # noqa: F821
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Any					       =				kwargs.pop("""do_lower_case"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: List[Any]					       =				tokenizer.do_lower_case if do_lower_case is None else do_lower_case
        SCREAMING_SNAKE_CASE	: Any					       =				kwargs.pop("""cls_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Any					       =				tokenizer.cls_token_id if cls_token_id is None else cls_token_id
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				kwargs.pop("""sep_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: List[Any]					       =				tokenizer.sep_token_id if sep_token_id is None else sep_token_id
        SCREAMING_SNAKE_CASE	: int					       =				kwargs.pop("""pad_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Any					       =				tokenizer.pad_token_id if pad_token_id is None else pad_token_id

        SCREAMING_SNAKE_CASE	: List[str]					       =				tokenizer.get_vocab()
        SCREAMING_SNAKE_CASE	: List[str]					       =				sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,			key=lambda UpperCAmelCase__						: x[1]						)
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				[entry[0] for entry in vocab]
        return cls(
            vocab_list=UpperCAmelCase__			,			do_lower_case=UpperCAmelCase__			,			cls_token_id=UpperCAmelCase__			,			sep_token_id=UpperCAmelCase__			,			pad_token_id=UpperCAmelCase__			,			**UpperCAmelCase__			,			)


  @classmethod
  def 	_lowercase (     cls      :	str			,			UpperCAmelCase__      :	Union[str, os.PathLike]			,			*UpperCAmelCase__      :	Dict			,			**UpperCAmelCase__      :	int						)   ->Any:
        """simple docstring"""







        try:
              SCREAMING_SNAKE_CASE	: List[str]					       =				BertTokenizer.from_pretrained(UpperCAmelCase__			,			*UpperCAmelCase__			,			**UpperCAmelCase__						)
        except:  # noqa: E722
              from .tokenization_bert_fast import BertTokenizerFast

              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				BertTokenizerFast.from_pretrained(UpperCAmelCase__			,			*UpperCAmelCase__			,			**UpperCAmelCase__						)
        return cls.from_tokenizer(UpperCAmelCase__			,			**UpperCAmelCase__						)


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	List[str]						)   ->List[str]:
        """simple docstring"""







        if self.do_lower_case:
              SCREAMING_SNAKE_CASE	: Optional[Any]					       =				case_fold_utfa(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: int					       =				self.tf_tokenizer.tokenize(UpperCAmelCase__						)
        return tokens.merge_dims(1			,			-1						)


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	Any			,			UpperCAmelCase__      :	Dict=None			,			UpperCAmelCase__      :	Optional[int]=None			,			UpperCAmelCase__      :	Optional[Any]=None			,			UpperCAmelCase__      :	Union[str, Any]=None			,			UpperCAmelCase__      :	Dict=None			,			UpperCAmelCase__      :	int=None			,			UpperCAmelCase__      :	Tuple=None			,			)   ->Any:
        """simple docstring"""







        if padding is None:
              SCREAMING_SNAKE_CASE	: int					       =				self.padding
        if padding not in ("longest", "max_length"):
              raise ValueError("""Padding must be either 'longest' or 'max_length'!"""						)
        if max_length is not None and text_pair is not None:
              # Because we have to instantiate a Trimmer to do it properly
              raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""						)
        if max_length is None:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				self.max_length
        if truncation is None:
              SCREAMING_SNAKE_CASE	: str					       =				self.truncation
        if pad_to_multiple_of is None:
              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.pad_to_multiple_of
        if return_token_type_ids is None:
              SCREAMING_SNAKE_CASE	: Any					       =				self.return_token_type_ids
        if return_attention_mask is None:
              SCREAMING_SNAKE_CASE	: int					       =				self.return_attention_mask
        if not isinstance(UpperCAmelCase__			,			tf.Tensor						):
              SCREAMING_SNAKE_CASE	: str					       =				tf.convert_to_tensor(UpperCAmelCase__						)
        if text_pair is not None and not isinstance(UpperCAmelCase__			,			tf.Tensor						):
              SCREAMING_SNAKE_CASE	: int					       =				tf.convert_to_tensor(UpperCAmelCase__						)
        if text_pair is not None:
              if text.shape.rank > 1:
                    raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""						)
              if text_pair.shape.rank > 1:
                    raise ValueError("""text_pair should not be multidimensional!"""						)
        if text.shape.rank == 2:
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Dict					       =				text[:, 0], text[:, 1]
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				self.unpaired_tokenize(UpperCAmelCase__						)
        if text_pair is None:  # Unpaired text
              if truncation:
                    SCREAMING_SNAKE_CASE	: int					       =				text[:, : max_length - 2]  # Allow room for special tokens
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: List[str]					       =				combine_segments(
                  (text,)			,			start_of_sequence_id=self.cls_token_id			,			end_of_segment_id=self.sep_token_id						)
        else:  # Paired text
              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.unpaired_tokenize(UpperCAmelCase__						)
              if truncation:
                    SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Any					       =				self.paired_trimmer.trim([text, text_pair]						)
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: str					       =				combine_segments(
                  (text, text_pair)			,			start_of_sequence_id=self.cls_token_id			,			end_of_segment_id=self.sep_token_id						)
        if padding == "longest":
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				input_ids.bounding_shape(axis=1						)
              if pad_to_multiple_of is not None:
                    # No ceiling division in tensorflow, so we negate floordiv instead
                    SCREAMING_SNAKE_CASE	: Any					       =				pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,			UpperCAmelCase__						))
        else:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				max_length

        SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Tuple					       =				pad_model_inputs(UpperCAmelCase__			,			max_seq_length=UpperCAmelCase__			,			pad_value=self.pad_token_id						)
        SCREAMING_SNAKE_CASE	: Any					       =				{"""input_ids""": input_ids}
        if return_attention_mask:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				attention_mask
        if return_token_type_ids:
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Optional[Any]					       =				pad_model_inputs(
                  UpperCAmelCase__			,			max_seq_length=UpperCAmelCase__			,			pad_value=self.pad_token_id						)
              SCREAMING_SNAKE_CASE	: List[str]					       =				token_type_ids
        return output




  def 	_lowercase (     self      :	Optional[int]						)   ->Tuple:
        """simple docstring"""







        return {
            "vocab_list": self.vocab_list,
            "do_lower_case": self.do_lower_case,
            "cls_token_id": self.cls_token_id,
            "sep_token_id": self.sep_token_id,
            "pad_token_id": self.pad_token_id,
        }




