




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import EfficientNetImageProcessor







class 			snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						def __init__(			self				,  __lowercase				,  __lowercase=1_3				,  __lowercase=3				,  __lowercase=1_8				,  __lowercase=3_0				,  __lowercase=4_0_0				,  __lowercase=True				,  __lowercase=None				,  __lowercase=True				,  __lowercase=[0.5, 0.5, 0.5]				,  __lowercase=[0.5, 0.5, 0.5]				,  )   ->      Any:
										lowerCAmelCase_	: Optional[int]      			=			size if size is not None else {'''height''': 1_8, '''width''': 1_8}
										lowerCAmelCase_	: Dict      			=			parent
										lowerCAmelCase_	: int      			=			batch_size
										lowerCAmelCase_	: List[Any]      			=			num_channels
										lowerCAmelCase_	: Optional[int]      			=			image_size
										lowerCAmelCase_	: str      			=			min_resolution
										lowerCAmelCase_	: Optional[Any]      			=			max_resolution
										lowerCAmelCase_	: Dict      			=			do_resize
										lowerCAmelCase_	: Tuple      			=			size
										lowerCAmelCase_	: str      			=			do_normalize
										lowerCAmelCase_	: int      			=			image_mean
										lowerCAmelCase_	: Dict      			=			image_std




						def   lowercase_	(			self							)   ->      Dict:
										return {
										    "image_mean": self.image_mean,
										    "image_std": self.image_std,
										    "do_normalize": self.do_normalize,
										    "do_resize": self.do_resize,
										    "size": self.size,
										}







@require_torch
@require_vision
class 			snake_case__(							UpperCAmelCase__,  unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__ :					List[Any]   =							EfficientNetImageProcessor if is_vision_available() else None




						def   lowercase_	(			self							)   ->      Dict:
										lowerCAmelCase_	: List[Any]      			=			EfficientNetImageProcessorTester(self							)




						@property
						def   lowercase_	(			self							)   ->      str:
										return self.image_processor_tester.prepare_image_processor_dict()




						def   lowercase_	(			self							)   ->      Tuple:
										lowerCAmelCase_	: str      			=			self.image_processing_class(**self.image_processor_dict							)
										self.assertTrue(hasattr(__lowercase				,  '''image_mean'''							)							)
										self.assertTrue(hasattr(__lowercase				,  '''image_std'''							)							)
										self.assertTrue(hasattr(__lowercase				,  '''do_normalize'''							)							)
										self.assertTrue(hasattr(__lowercase				,  '''do_resize'''							)							)
										self.assertTrue(hasattr(__lowercase				,  '''size'''							)							)




						def   lowercase_	(			self							)   ->      List[Any]:
										lowerCAmelCase_	: Tuple      			=			self.image_processing_class.from_dict(self.image_processor_dict							)
										self.assertEqual(image_processor.size				,  {'''height''': 1_8, '''width''': 1_8}							)

										lowerCAmelCase_	: Optional[Any]      			=			self.image_processing_class.from_dict(self.image_processor_dict				,  size=4_2							)
										self.assertEqual(image_processor.size				,  {'''height''': 4_2, '''width''': 4_2}							)




						def   lowercase_	(			self							)   ->      Optional[int]:
										# Initialize image_processing
										lowerCAmelCase_	: Any      			=			self.image_processing_class(**self.image_processor_dict							)
										# create random PIL images
										lowerCAmelCase_	: Optional[Any]      			=			prepare_image_inputs(self.image_processor_tester				,  equal_resolution=__lowercase							)
										for image in image_inputs:
														self.assertIsInstance(__lowercase				,  Image.Image							)

										# Test not batched input
										lowerCAmelCase_	: Any      			=			image_processing(image_inputs[0]				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )

										# Test batched
										lowerCAmelCase_	: Dict      			=			image_processing(__lowercase				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )




						def   lowercase_	(			self							)   ->      Any:
										# Initialize image_processing
										lowerCAmelCase_	: List[Any]      			=			self.image_processing_class(**self.image_processor_dict							)
										# create random numpy tensors
										lowerCAmelCase_	: int      			=			prepare_image_inputs(self.image_processor_tester				,  equal_resolution=__lowercase				,  numpify=__lowercase							)
										for image in image_inputs:
														self.assertIsInstance(__lowercase				,  np.ndarray							)

										# Test not batched input
										lowerCAmelCase_	: Union[str, Any]      			=			image_processing(image_inputs[0]				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )

										# Test batched
										lowerCAmelCase_	: Tuple      			=			image_processing(__lowercase				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )




						def   lowercase_	(			self							)   ->      Any:
										# Initialize image_processing
										lowerCAmelCase_	: Optional[Any]      			=			self.image_processing_class(**self.image_processor_dict							)
										# create random PyTorch tensors
										lowerCAmelCase_	: Any      			=			prepare_image_inputs(self.image_processor_tester				,  equal_resolution=__lowercase				,  torchify=__lowercase							)
										for image in image_inputs:
														self.assertIsInstance(__lowercase				,  torch.Tensor							)

										# Test not batched input
										lowerCAmelCase_	: Any      			=			image_processing(image_inputs[0]				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )

										# Test batched
										lowerCAmelCase_	: List[str]      			=			image_processing(__lowercase				,  return_tensors='''pt'''							).pixel_values
										self.assertEqual(
										    encoded_images.shape				,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    )				,  )



from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase     :					List[str]       =logging.get_logger(__name__)







class 			snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__ :					List[str]   =							["""input_ids""", """attention_mask"""]




						def __init__(			self				,  __lowercase="[PAD]"				,  __lowercase="[BOS]"				,  __lowercase="[EOS]"				,  __lowercase="[MASK]"				,  __lowercase="[CLS]"				,  __lowercase="[SEP]"				,  __lowercase=2_0_4_8				,  **__lowercase				,  )   ->      None:
										lowerCAmelCase_	: Optional[int]      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else pad_token
										lowerCAmelCase_	: str      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else bos_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else eos_token
										lowerCAmelCase_	: List[str]      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else mask_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else cls_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else sep_token

										super().__init__(
										    pad_token=__lowercase				,  bos_token=__lowercase				,  eos_token=__lowercase				,  mask_token=__lowercase				,  cls_token=__lowercase				,  sep_token=__lowercase				,  model_max_length=__lowercase				,  **__lowercase				,  )

										lowerCAmelCase_	: Dict      			=			2**8  # utf is 8 bits

										# define special tokens dict
										lowerCAmelCase_	: Dict[str, int]      			=			{
										    self.pad_token: 0,
										    self.bos_token: 1,
										    self.eos_token: 2,
										    self.mask_token: 3,
										    self.cls_token: 4,
										    self.sep_token: 5,
										}
										lowerCAmelCase_	: List[Any]      			=			len(self.special_tokens_encoder							)
										lowerCAmelCase_	: Dict[int, str]      			=			{v: k for k, v in self.special_tokens_encoder.items()}




						def   lowercase_	(			self							)   ->      Dict[str, int]:
										lowerCAmelCase_	: int      			=			self.special_tokens_encoder.copy()
										vocab.update(self.added_tokens_encoder							)
										for i in range(self._utf_vocab_size							):
														lowerCAmelCase_	: int      			=			chr(__lowercase							)
														lowerCAmelCase_	: Optional[Any]      			=			i + len(self.special_tokens_encoder							)
										return vocab




						@property
						def   lowercase_	(			self							)   ->      str:
										return self._utf_vocab_size + self._num_special_tokens




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None				,  __lowercase = False							)   ->      List[int]:
										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=__lowercase				,  token_ids_a=__lowercase				,  already_has_special_tokens=__lowercase							)

										# normal case: some special tokens
										if token_ids_a is None:
														return [1] + [0] * len(__lowercase							) + [1]
										return [1] + ([0] * len(__lowercase							)) + [1] + ([0] * len(__lowercase							)) + [1]




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None							)   ->      List[int]:
										if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										else:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]




						def   lowercase_	(			self				,  __lowercase							)   ->      List[str]:
										lowerCAmelCase_	: Optional[int]      			=			[chr(__lowercase							) for i in text.encode('''utf-8'''							)]
										return tokens




						def   lowercase_	(			self				,  __lowercase							)   ->      Tuple:
										if token in self.special_tokens_encoder:
														lowerCAmelCase_	: Optional[int]      			=			self.special_tokens_encoder[token]
										elif token in self.added_tokens_encoder:
														lowerCAmelCase_	: Any      			=			self.added_tokens_encoder[token]
										elif len(__lowercase							) != 1:
														lowerCAmelCase_	: Any      			=			self.unk_token_id
										else:
														lowerCAmelCase_	: List[Any]      			=			ord(__lowercase							) + self._num_special_tokens
										return token_id




						def   lowercase_	(			self				,  __lowercase							)   ->      Optional[int]:
										if index in self.special_tokens_decoder:
														lowerCAmelCase_	: List[Any]      			=			self.special_tokens_decoder[index]
										elif index in self.added_tokens_decoder:
														lowerCAmelCase_	: List[str]      			=			self.added_tokens_decoder[index]
										else:
														lowerCAmelCase_	: str      			=			chr(index - self._num_special_tokens							)
										return token




						def   lowercase_	(			self				,  __lowercase							)   ->      Optional[int]:
										lowerCAmelCase_	: Tuple      			=			B''''''
										for token in tokens:
														if token in self.special_tokens_decoder:
																		lowerCAmelCase_	: int      			=			self.special_tokens_decoder[token].encode('''utf-8'''							)
														elif token in self.added_tokens_decoder:
																		lowerCAmelCase_	: Tuple      			=			self.special_tokens_decoder[token].encode('''utf-8'''							)
														elif token in self.special_tokens_encoder:
																		lowerCAmelCase_	: Any      			=			token.encode('''utf-8'''							)
														elif token in self.added_tokens_encoder:
																		lowerCAmelCase_	: Tuple      			=			token.encode('''utf-8'''							)
														else:
																		lowerCAmelCase_	: str      			=			bytes([ord(__lowercase							)]							)
														bstring += tok_string
										lowerCAmelCase_	: str      			=			bstring.decode('''utf-8'''				,  errors='''replace'''							)
										return string




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None							)   ->      Tuple[str]:
										return ()