




'''simple docstring'''
import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer






class 	SCREAMING_SNAKE_CASE(    tf.keras.layers.Layer					):

       def __init__(   self ,		lowerCamelCase__ ,		lowerCamelCase__ ,		lowerCamelCase__ = None ,		lowerCamelCase__ = None ,		lowerCamelCase__ = None ,		lowerCamelCase__ = "longest" ,		lowerCamelCase__ = True ,		lowerCamelCase__ = 512 ,		lowerCamelCase__ = None ,		lowerCamelCase__ = True ,		lowerCamelCase__ = True ,		lowerCamelCase__ = True ,		**lowerCamelCase__ ,		)   -> Optional[int]:

           """simple docstring"""



           super().__init__()
           if use_fast_bert_tokenizer:
               __lowercase          =						FastBertTokenizer(
                   lowerCamelCase__ ,		token_out_type=tf.intaa ,		lower_case_nfd_strip_accents=lowerCamelCase__ ,		**lowerCamelCase__					)
           else:
               __lowercase          =						tf.lookup.StaticVocabularyTable(
                   tf.lookup.KeyValueTensorInitializer(
                       keys=lowerCamelCase__ ,		key_dtype=tf.string ,		values=tf.range(tf.size(lowerCamelCase__ ,		out_type=tf.intaa					) ,		dtype=tf.intaa					) ,		value_dtype=tf.intaa ,		) ,		num_oov_buckets=1 ,		)
               __lowercase          =						BertTokenizerLayer(
                   lowerCamelCase__ ,		token_out_type=tf.intaa ,		lower_case=lowerCamelCase__ ,		**lowerCamelCase__					)

           __lowercase          =						vocab_list
           __lowercase          =						do_lower_case
           __lowercase          =						cls_token_id or vocab_list.index("""[CLS]"""					)
           __lowercase          =						sep_token_id or vocab_list.index("""[SEP]"""					)
           __lowercase          =						pad_token_id or vocab_list.index("""[PAD]"""					)
           __lowercase          =						ShrinkLongestTrimmer(max_length - 3 ,		axis=1					)  # Allow room for special tokens
           __lowercase          =						max_length
           __lowercase          =						padding
           __lowercase          =						truncation
           __lowercase          =						pad_to_multiple_of
           __lowercase          =						return_token_type_ids
           __lowercase          =						return_attention_mask

       @classmethod
       def    snake_case__      (   cls ,		lowerCamelCase__ ,		**lowerCamelCase__					)   -> str:  # noqa: F821

           """simple docstring"""



           __lowercase          =						kwargs.pop("""do_lower_case""" ,		lowerCamelCase__					)
           __lowercase          =						tokenizer.do_lower_case if do_lower_case is None else do_lower_case
           __lowercase          =						kwargs.pop("""cls_token_id""" ,		lowerCamelCase__					)
           __lowercase          =						tokenizer.cls_token_id if cls_token_id is None else cls_token_id
           __lowercase          =						kwargs.pop("""sep_token_id""" ,		lowerCamelCase__					)
           __lowercase          =						tokenizer.sep_token_id if sep_token_id is None else sep_token_id
           __lowercase          =						kwargs.pop("""pad_token_id""" ,		lowerCamelCase__					)
           __lowercase          =						tokenizer.pad_token_id if pad_token_id is None else pad_token_id

           __lowercase          =						tokenizer.get_vocab()
           __lowercase          =						sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,		key=lambda lowerCamelCase__					: x[1]					)
           __lowercase          =						[entry[0] for entry in vocab]
           return cls(
               vocab_list=lowerCamelCase__ ,		do_lower_case=lowerCamelCase__ ,		cls_token_id=lowerCamelCase__ ,		sep_token_id=lowerCamelCase__ ,		pad_token_id=lowerCamelCase__ ,		**lowerCamelCase__ ,		)

       @classmethod
       def    snake_case__      (   cls ,		lowerCamelCase__ ,		*lowerCamelCase__ ,		**lowerCamelCase__					)   -> str:

           """simple docstring"""



           try:
               __lowercase          =						BertTokenizer.from_pretrained(lowerCamelCase__ ,		*lowerCamelCase__ ,		**lowerCamelCase__					)
           except:  # noqa: E722
               from .tokenization_bert_fast import BertTokenizerFast

               __lowercase          =						BertTokenizerFast.from_pretrained(lowerCamelCase__ ,		*lowerCamelCase__ ,		**lowerCamelCase__					)
           return cls.from_tokenizer(lowerCamelCase__ ,		**lowerCamelCase__					)

       def    snake_case__      (   self ,		lowerCamelCase__					)   -> Tuple:

           """simple docstring"""



           if self.do_lower_case:
               __lowercase          =						case_fold_utfa(lowerCamelCase__					)
           __lowercase          =						self.tf_tokenizer.tokenize(lowerCamelCase__					)
           return tokens.merge_dims(1 ,		-1					)

       def    snake_case__      (   self ,		lowerCamelCase__ ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		lowerCamelCase__=None ,		)   -> int:

           """simple docstring"""



           if padding is None:
               __lowercase          =						self.padding
           if padding not in ("longest", "max_length"):
               raise ValueError("""Padding must be either 'longest' or 'max_length'!"""					)
           if max_length is not None and text_pair is not None:
               # Because we have to instantiate a Trimmer to do it properly
               raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""					)
           if max_length is None:
               __lowercase          =						self.max_length
           if truncation is None:
               __lowercase          =						self.truncation
           if pad_to_multiple_of is None:
               __lowercase          =						self.pad_to_multiple_of
           if return_token_type_ids is None:
               __lowercase          =						self.return_token_type_ids
           if return_attention_mask is None:
               __lowercase          =						self.return_attention_mask
           if not isinstance(lowerCamelCase__ ,		tf.Tensor					):
               __lowercase          =						tf.convert_to_tensor(lowerCamelCase__					)
           if text_pair is not None and not isinstance(lowerCamelCase__ ,		tf.Tensor					):
               __lowercase          =						tf.convert_to_tensor(lowerCamelCase__					)
           if text_pair is not None:
               if text.shape.rank > 1:
                   raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""					)
               if text_pair.shape.rank > 1:
                   raise ValueError("""text_pair should not be multidimensional!"""					)
           if text.shape.rank == 2:
               __lowercase    ,__lowercase          =						text[:, 0], text[:, 1]
           __lowercase          =						self.unpaired_tokenize(lowerCamelCase__					)
           if text_pair is None:  # Unpaired text
               if truncation:
                   __lowercase          =						text[:, : max_length - 2]  # Allow room for special tokens
               __lowercase    ,__lowercase          =						combine_segments(
                   (text,) ,		start_of_sequence_id=self.cls_token_id ,		end_of_segment_id=self.sep_token_id					)
           else:  # Paired text
               __lowercase          =						self.unpaired_tokenize(lowerCamelCase__					)
               if truncation:
                   __lowercase    ,__lowercase          =						self.paired_trimmer.trim([text, text_pair]					)
               __lowercase    ,__lowercase          =						combine_segments(
                   (text, text_pair) ,		start_of_sequence_id=self.cls_token_id ,		end_of_segment_id=self.sep_token_id					)
           if padding == "longest":
               __lowercase          =						input_ids.bounding_shape(axis=1					)
               if pad_to_multiple_of is not None:
                   # No ceiling division in tensorflow, so we negate floordiv instead
                   __lowercase          =						pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,		lowerCamelCase__					))
           else:
               __lowercase          =						max_length

           __lowercase    ,__lowercase          =						pad_model_inputs(lowerCamelCase__ ,		max_seq_length=lowerCamelCase__ ,		pad_value=self.pad_token_id					)
           __lowercase          =						{"""input_ids""": input_ids}
           if return_attention_mask:
               __lowercase          =						attention_mask
           if return_token_type_ids:
               __lowercase    ,__lowercase          =						pad_model_inputs(
                   lowerCamelCase__ ,		max_seq_length=lowerCamelCase__ ,		pad_value=self.pad_token_id					)
               __lowercase          =						token_type_ids
           return output




       def    snake_case__      (   self					)   -> List[str]:

           """simple docstring"""



           return {
               "vocab_list": self.vocab_list,
               "do_lower_case": self.do_lower_case,
               "cls_token_id": self.cls_token_id,
               "sep_token_id": self.sep_token_id,
               "pad_token_id": self.pad_token_id,
           }










'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A							:				Tuple			     =	{
    """configuration_dpr""": ["""DPR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DPRConfig"""],
    """tokenization_dpr""": [
        """DPRContextEncoderTokenizer""",
        """DPRQuestionEncoderTokenizer""",
        """DPRReaderOutput""",
        """DPRReaderTokenizer""",
    ],
}


try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  A							:				str			     =	[
      """DPRContextEncoderTokenizerFast""",
      """DPRQuestionEncoderTokenizerFast""",
      """DPRReaderTokenizerFast""",
  ]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  A							:				Tuple			     =	[
      """DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """DPRContextEncoder""",
      """DPRPretrainedContextEncoder""",
      """DPRPreTrainedModel""",
      """DPRPretrainedQuestionEncoder""",
      """DPRPretrainedReader""",
      """DPRQuestionEncoder""",
      """DPRReader""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  A							:				Tuple			     =	[
      """TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFDPRContextEncoder""",
      """TFDPRPretrainedContextEncoder""",
      """TFDPRPretrainedQuestionEncoder""",
      """TFDPRPretrainedReader""",
      """TFDPRQuestionEncoder""",
      """TFDPRReader""",
  ]


if TYPE_CHECKING:
  from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
  from .tokenization_dpr import (
      DPRContextEncoderTokenizer,
      DPRQuestionEncoderTokenizer,
      DPRReaderOutput,
      DPRReaderTokenizer,
  )

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_dpr_fast import (
        DPRContextEncoderTokenizerFast,
        DPRQuestionEncoderTokenizerFast,
        DPRReaderTokenizerFast,
    )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_dpr import (
        DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
        DPRContextEncoder,
        DPRPretrainedContextEncoder,
        DPRPreTrainedModel,
        DPRPretrainedQuestionEncoder,
        DPRPretrainedReader,
        DPRQuestionEncoder,
        DPRReader,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_dpr import (
        TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDPRContextEncoder,
        TFDPRPretrainedContextEncoder,
        TFDPRPretrainedQuestionEncoder,
        TFDPRPretrainedReader,
        TFDPRQuestionEncoder,
        TFDPRReader,
    )

else:
  import sys

  A							:				int			     =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




