

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import MobileNetVaImageProcessor







class     a__    (				unittest.TestCase    ):







  """simple docstring"""




  def __init__(     self      :	Any			,			UpperCAmelCase__      :	Union[str, Any]			,			UpperCAmelCase__      :	Union[str, Any]=7			,			UpperCAmelCase__      :	Tuple=3			,			UpperCAmelCase__      :	List[Any]=1_8			,			UpperCAmelCase__      :	List[Any]=3_0			,			UpperCAmelCase__      :	Union[str, Any]=4_0_0			,			UpperCAmelCase__      :	int=True			,			UpperCAmelCase__      :	Optional[int]=None			,			UpperCAmelCase__      :	Optional[int]=True			,			UpperCAmelCase__      :	List[str]=None			,			)   ->int:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: str					       =				size if size is not None else {"""shortest_edge""": 2_0}
        SCREAMING_SNAKE_CASE	: str					       =				crop_size if crop_size is not None else {"""height""": 1_8, """width""": 1_8}
        SCREAMING_SNAKE_CASE	: List[Any]					       =				parent
        SCREAMING_SNAKE_CASE	: List[Any]					       =				batch_size
        SCREAMING_SNAKE_CASE	: List[str]					       =				num_channels
        SCREAMING_SNAKE_CASE	: Dict					       =				image_size
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				min_resolution
        SCREAMING_SNAKE_CASE	: List[Any]					       =				max_resolution
        SCREAMING_SNAKE_CASE	: Dict					       =				do_resize
        SCREAMING_SNAKE_CASE	: List[str]					       =				size
        SCREAMING_SNAKE_CASE	: Tuple					       =				do_center_crop
        SCREAMING_SNAKE_CASE	: str					       =				crop_size




  def 	_lowercase (     self      :	Dict						)   ->List[Any]:
        """simple docstring"""







        return {
            "do_resize": self.do_resize,
            "size": self.size,
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
        }







@require_torch
@require_vision
class     a__    (				UpperCAmelCase		,     unittest.TestCase    ):







  """simple docstring"""




  UpperCAmelCase__ :	Union[str, Any]         =MobileNetVaImageProcessor if is_vision_available() else None


  def 	_lowercase (     self      :	List[str]						)   ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				MobileNetVaImageProcessingTester(self						)


  @property
  def 	_lowercase (     self      :	Optional[Any]						)   ->List[str]:
        """simple docstring"""







        return self.image_processor_tester.prepare_image_processor_dict()


  def 	_lowercase (     self      :	List[str]						)   ->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Optional[int]					       =				self.image_processing_class(**self.image_processor_dict						)
        self.assertTrue(hasattr(UpperCAmelCase__			,			"""do_resize"""						)						)
        self.assertTrue(hasattr(UpperCAmelCase__			,			"""size"""						)						)
        self.assertTrue(hasattr(UpperCAmelCase__			,			"""do_center_crop"""						)						)
        self.assertTrue(hasattr(UpperCAmelCase__			,			"""center_crop"""						)						)


  def 	_lowercase (     self      :	List[Any]						)   ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.image_processing_class.from_dict(self.image_processor_dict						)
        self.assertEqual(image_processor.size			,			{"""shortest_edge""": 2_0}						)
        self.assertEqual(image_processor.crop_size			,			{"""height""": 1_8, """width""": 1_8}						)

        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.image_processing_class.from_dict(self.image_processor_dict			,			size=4_2			,			crop_size=8_4						)
        self.assertEqual(image_processor.size			,			{"""shortest_edge""": 4_2}						)
        self.assertEqual(image_processor.crop_size			,			{"""height""": 8_4, """width""": 8_4}						)


  def 	_lowercase (     self      :	Optional[int]						)   ->Optional[Any]:
        """simple docstring"""







        pass


  def 	_lowercase (     self      :	List[Any]						)   ->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.image_processing_class(**self.image_processor_dict						)
        # create random PIL images
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				prepare_image_inputs(self.image_processor_tester			,			equal_resolution=UpperCAmelCase__						)
        for image in image_inputs:
              self.assertIsInstance(UpperCAmelCase__			,			Image.Image						)

        # Test not batched input
        SCREAMING_SNAKE_CASE	: List[Any]					       =				image_processing(image_inputs[0]			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)

        # Test batched
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				image_processing(UpperCAmelCase__			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)


  def 	_lowercase (     self      :	Union[str, Any]						)   ->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Any					       =				self.image_processing_class(**self.image_processor_dict						)
        # create random numpy tensors
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				prepare_image_inputs(self.image_processor_tester			,			equal_resolution=UpperCAmelCase__			,			numpify=UpperCAmelCase__						)
        for image in image_inputs:
              self.assertIsInstance(UpperCAmelCase__			,			np.ndarray						)

        # Test not batched input
        SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				image_processing(image_inputs[0]			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)

        # Test batched
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				image_processing(UpperCAmelCase__			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)




  def 	_lowercase (     self      :	Tuple						)   ->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Any					       =				self.image_processing_class(**self.image_processor_dict						)
        # create random PyTorch tensors
        SCREAMING_SNAKE_CASE	: Any					       =				prepare_image_inputs(self.image_processor_tester			,			equal_resolution=UpperCAmelCase__			,			torchify=UpperCAmelCase__						)
        for image in image_inputs:
              self.assertIsInstance(UpperCAmelCase__			,			torch.Tensor						)

        # Test not batched input
        SCREAMING_SNAKE_CASE	: str					       =				image_processing(image_inputs[0]			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)

        # Test batched
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				image_processing(UpperCAmelCase__			,			return_tensors="""pt"""						).pixel_values
        self.assertEqual(
            encoded_images.shape			,			(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["""height"""],
                self.image_processor_tester.crop_size["""width"""],
            )			,			)







import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer







class     a__    (				tf.keras.layers.Layer    ):







  """simple docstring"""




  def __init__(     self      :	str			,			UpperCAmelCase__      :	List			,			UpperCAmelCase__      :	bool			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	str = "longest"			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	int = 5_1_2			,			UpperCAmelCase__      :	int = None			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	bool = True			,			UpperCAmelCase__      :	bool = True			,			**UpperCAmelCase__      :	Optional[int]			,			)   ->Union[str, Any]:
        """simple docstring"""







        super().__init__()
        if use_fast_bert_tokenizer:
              SCREAMING_SNAKE_CASE	: Any					       =				FastBertTokenizer(
                  UpperCAmelCase__			,			token_out_type=tf.intaa			,			lower_case_nfd_strip_accents=UpperCAmelCase__			,			**UpperCAmelCase__						)
        else:
              SCREAMING_SNAKE_CASE	: Any					       =				tf.lookup.StaticVocabularyTable(
                  tf.lookup.KeyValueTensorInitializer(
                      keys=UpperCAmelCase__			,			key_dtype=tf.string			,			values=tf.range(tf.size(UpperCAmelCase__			,			out_type=tf.intaa						)			,			dtype=tf.intaa						)			,			value_dtype=tf.intaa			,			)			,			num_oov_buckets=1			,			)
              SCREAMING_SNAKE_CASE	: Dict					       =				BertTokenizerLayer(
                  UpperCAmelCase__			,			token_out_type=tf.intaa			,			lower_case=UpperCAmelCase__			,			**UpperCAmelCase__						)

        SCREAMING_SNAKE_CASE	: Any					       =				vocab_list
        SCREAMING_SNAKE_CASE	: int					       =				do_lower_case
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				cls_token_id or vocab_list.index("""[CLS]"""						)
        SCREAMING_SNAKE_CASE	: int					       =				sep_token_id or vocab_list.index("""[SEP]"""						)
        SCREAMING_SNAKE_CASE	: int					       =				pad_token_id or vocab_list.index("""[PAD]"""						)
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				ShrinkLongestTrimmer(max_length - 3			,			axis=1						)  # Allow room for special tokens
        SCREAMING_SNAKE_CASE	: Tuple					       =				max_length
        SCREAMING_SNAKE_CASE	: List[str]					       =				padding
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				truncation
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				pad_to_multiple_of
        SCREAMING_SNAKE_CASE	: Dict					       =				return_token_type_ids
        SCREAMING_SNAKE_CASE	: str					       =				return_attention_mask


  @classmethod
  def 	_lowercase (     cls      :	Optional[int]			,			UpperCAmelCase__      :	"PreTrainedTokenizerBase"			,			**UpperCAmelCase__      :	Tuple						)   ->str:  # noqa: F821
        """simple docstring"""







        SCREAMING_SNAKE_CASE	: Any					       =				kwargs.pop("""do_lower_case"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: List[Any]					       =				tokenizer.do_lower_case if do_lower_case is None else do_lower_case
        SCREAMING_SNAKE_CASE	: Any					       =				kwargs.pop("""cls_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Any					       =				tokenizer.cls_token_id if cls_token_id is None else cls_token_id
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				kwargs.pop("""sep_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: List[Any]					       =				tokenizer.sep_token_id if sep_token_id is None else sep_token_id
        SCREAMING_SNAKE_CASE	: int					       =				kwargs.pop("""pad_token_id"""			,			UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: Any					       =				tokenizer.pad_token_id if pad_token_id is None else pad_token_id

        SCREAMING_SNAKE_CASE	: List[str]					       =				tokenizer.get_vocab()
        SCREAMING_SNAKE_CASE	: List[str]					       =				sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,			key=lambda UpperCAmelCase__						: x[1]						)
        SCREAMING_SNAKE_CASE	: Optional[Any]					       =				[entry[0] for entry in vocab]
        return cls(
            vocab_list=UpperCAmelCase__			,			do_lower_case=UpperCAmelCase__			,			cls_token_id=UpperCAmelCase__			,			sep_token_id=UpperCAmelCase__			,			pad_token_id=UpperCAmelCase__			,			**UpperCAmelCase__			,			)


  @classmethod
  def 	_lowercase (     cls      :	str			,			UpperCAmelCase__      :	Union[str, os.PathLike]			,			*UpperCAmelCase__      :	Dict			,			**UpperCAmelCase__      :	int						)   ->Any:
        """simple docstring"""







        try:
              SCREAMING_SNAKE_CASE	: List[str]					       =				BertTokenizer.from_pretrained(UpperCAmelCase__			,			*UpperCAmelCase__			,			**UpperCAmelCase__						)
        except:  # noqa: E722
              from .tokenization_bert_fast import BertTokenizerFast

              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				BertTokenizerFast.from_pretrained(UpperCAmelCase__			,			*UpperCAmelCase__			,			**UpperCAmelCase__						)
        return cls.from_tokenizer(UpperCAmelCase__			,			**UpperCAmelCase__						)


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	List[str]						)   ->List[str]:
        """simple docstring"""







        if self.do_lower_case:
              SCREAMING_SNAKE_CASE	: Optional[Any]					       =				case_fold_utfa(UpperCAmelCase__						)
        SCREAMING_SNAKE_CASE	: int					       =				self.tf_tokenizer.tokenize(UpperCAmelCase__						)
        return tokens.merge_dims(1			,			-1						)


  def 	_lowercase (     self      :	Tuple			,			UpperCAmelCase__      :	Any			,			UpperCAmelCase__      :	Dict=None			,			UpperCAmelCase__      :	Optional[int]=None			,			UpperCAmelCase__      :	Optional[Any]=None			,			UpperCAmelCase__      :	Union[str, Any]=None			,			UpperCAmelCase__      :	Dict=None			,			UpperCAmelCase__      :	int=None			,			UpperCAmelCase__      :	Tuple=None			,			)   ->Any:
        """simple docstring"""







        if padding is None:
              SCREAMING_SNAKE_CASE	: int					       =				self.padding
        if padding not in ("longest", "max_length"):
              raise ValueError("""Padding must be either 'longest' or 'max_length'!"""						)
        if max_length is not None and text_pair is not None:
              # Because we have to instantiate a Trimmer to do it properly
              raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""						)
        if max_length is None:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				self.max_length
        if truncation is None:
              SCREAMING_SNAKE_CASE	: str					       =				self.truncation
        if pad_to_multiple_of is None:
              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.pad_to_multiple_of
        if return_token_type_ids is None:
              SCREAMING_SNAKE_CASE	: Any					       =				self.return_token_type_ids
        if return_attention_mask is None:
              SCREAMING_SNAKE_CASE	: int					       =				self.return_attention_mask
        if not isinstance(UpperCAmelCase__			,			tf.Tensor						):
              SCREAMING_SNAKE_CASE	: str					       =				tf.convert_to_tensor(UpperCAmelCase__						)
        if text_pair is not None and not isinstance(UpperCAmelCase__			,			tf.Tensor						):
              SCREAMING_SNAKE_CASE	: int					       =				tf.convert_to_tensor(UpperCAmelCase__						)
        if text_pair is not None:
              if text.shape.rank > 1:
                    raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""						)
              if text_pair.shape.rank > 1:
                    raise ValueError("""text_pair should not be multidimensional!"""						)
        if text.shape.rank == 2:
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Dict					       =				text[:, 0], text[:, 1]
        SCREAMING_SNAKE_CASE	: Optional[int]					       =				self.unpaired_tokenize(UpperCAmelCase__						)
        if text_pair is None:  # Unpaired text
              if truncation:
                    SCREAMING_SNAKE_CASE	: int					       =				text[:, : max_length - 2]  # Allow room for special tokens
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: List[str]					       =				combine_segments(
                  (text,)			,			start_of_sequence_id=self.cls_token_id			,			end_of_segment_id=self.sep_token_id						)
        else:  # Paired text
              SCREAMING_SNAKE_CASE	: Union[str, Any]					       =				self.unpaired_tokenize(UpperCAmelCase__						)
              if truncation:
                    SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Any					       =				self.paired_trimmer.trim([text, text_pair]						)
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: str					       =				combine_segments(
                  (text, text_pair)			,			start_of_sequence_id=self.cls_token_id			,			end_of_segment_id=self.sep_token_id						)
        if padding == "longest":
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				input_ids.bounding_shape(axis=1						)
              if pad_to_multiple_of is not None:
                    # No ceiling division in tensorflow, so we negate floordiv instead
                    SCREAMING_SNAKE_CASE	: Any					       =				pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,			UpperCAmelCase__						))
        else:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				max_length

        SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Tuple					       =				pad_model_inputs(UpperCAmelCase__			,			max_seq_length=UpperCAmelCase__			,			pad_value=self.pad_token_id						)
        SCREAMING_SNAKE_CASE	: Any					       =				{"""input_ids""": input_ids}
        if return_attention_mask:
              SCREAMING_SNAKE_CASE	: Optional[int]					       =				attention_mask
        if return_token_type_ids:
              SCREAMING_SNAKE_CASE       ,		SCREAMING_SNAKE_CASE	: Optional[Any]					       =				pad_model_inputs(
                  UpperCAmelCase__			,			max_seq_length=UpperCAmelCase__			,			pad_value=self.pad_token_id						)
              SCREAMING_SNAKE_CASE	: List[str]					       =				token_type_ids
        return output




  def 	_lowercase (     self      :	Optional[int]						)   ->Tuple:
        """simple docstring"""







        return {
            "vocab_list": self.vocab_list,
            "do_lower_case": self.do_lower_case,
            "cls_token_id": self.cls_token_id,
            "sep_token_id": self.sep_token_id,
            "pad_token_id": self.pad_token_id,
        }




