






"""simple docstring"""


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import CLIPImageProcessor


class  snake_case		(							unittest.TestCase						):


   '''simple docstring'''

   def __init__(     self			:					str,	_lowerCamelCase			:					List[Any],	_lowerCamelCase			:					Any=7,	_lowerCamelCase			:					Optional[Any]=3,	_lowerCamelCase			:					Optional[Any]=18,	_lowerCamelCase			:					Optional[Any]=30,	_lowerCamelCase			:					Any=4_00,	_lowerCamelCase			:					str=True,	_lowerCamelCase			:					Optional[int]=None,	_lowerCamelCase			:					Optional[int]=True,	_lowerCamelCase			:					List[Any]=None,	_lowerCamelCase			:					Union[str, Any]=True,	_lowerCamelCase			:					Any=[0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],	_lowerCamelCase			:					Any=[0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],	_lowerCamelCase			:					List[str]=True,	):

     '''simple docstring'''
     __A						      =			size if size is not None else {'''shortest_edge''': 20}
     __A						      =			crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
     __A						      =			parent
     __A						      =			batch_size
     __A						      =			num_channels
     __A						      =			image_size
     __A						      =			min_resolution
     __A						      =			max_resolution
     __A						      =			do_resize
     __A						      =			size
     __A						      =			do_center_crop
     __A						      =			crop_size
     __A						      =			do_normalize
     __A						      =			image_mean
     __A						      =			image_std
     __A						      =			do_convert_rgb


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     return {
         "do_resize": self.do_resize,
         "size": self.size,
         "do_center_crop": self.do_center_crop,
         "crop_size": self.crop_size,
         "do_normalize": self.do_normalize,
         "image_mean": self.image_mean,
         "image_std": self.image_std,
         "do_convert_rgb": self.do_convert_rgb,
     }






   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict,	_lowerCamelCase			:					Optional[int]=False,	_lowerCamelCase			:					Union[str, Any]=False,	_lowerCamelCase			:					str=False  ):

     '''simple docstring'''
     assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

     if equal_resolution:
       __A						      =			[]
       for i in range(self.batch_size  ):
         image_inputs.append(
             np.random.randint(
                 2_55,	size=(self.num_channels, self.max_resolution, self.max_resolution),	dtype=np.uinta  )  )
     else:
       __A						      =			[]
       for i in range(self.batch_size  ):
         __A      ,   __A						      =			np.random.choice(np.arange(self.min_resolution,	self.max_resolution  ),	2  )
         image_inputs.append(np.random.randint(2_55,	size=(self.num_channels, width, height),	dtype=np.uinta  )  )

     if not numpify and not torchify:
       # PIL expects the channel dimension as last dimension
       __A						      =			[Image.fromarray(np.moveaxis(_lowerCamelCase,	0,	-1  )  ) for x in image_inputs]

     if torchify:
       __A						      =			[torch.from_numpy(_lowerCamelCase  ) for x in image_inputs]

     return image_inputs





@require_torch
@require_vision
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     List[str]									=			CLIPImageProcessor if is_vision_available() else None


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     __A						      =			CLIPImageProcessingTester(self  )


   @property
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     return self.image_processor_tester.prepare_image_processor_dict()


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_resize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''size'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_center_crop'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''center_crop'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_normalize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''image_mean'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''image_std'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_convert_rgb'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     __A						      =			self.image_processing_class.from_dict(self.image_processor_dict  )
     self.assertEqual(image_processor.size,	{'''shortest_edge''': 20}  )
     self.assertEqual(image_processor.crop_size,	{'''height''': 18, '''width''': 18}  )

     __A						      =			self.image_processing_class.from_dict(self.image_processor_dict,	size=42,	crop_size=84  )
     self.assertEqual(image_processor.size,	{'''shortest_edge''': 42}  )
     self.assertEqual(image_processor.crop_size,	{'''height''': 84, '''width''': 84}  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     pass


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     # Initialize image_processing
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A						      =			self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,	Image.Image  )

     # Test not batched input
     __A						      =			image_processing(image_inputs[0],	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)

     # Test batched
     __A						      =			image_processing(_lowerCamelCase,	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     # Initialize image_processing
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     # create random numpy tensors
     __A						      =			self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCamelCase,	numpify=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,	np.ndarray  )

     # Test not batched input
     __A						      =			image_processing(image_inputs[0],	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)

     # Test batched
     __A						      =			image_processing(_lowerCamelCase,	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)






   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     # Initialize image_processing
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     # create random PyTorch tensors
     __A						      =			self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCamelCase,	torchify=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,	torch.Tensor  )

     # Test not batched input
     __A						      =			image_processing(image_inputs[0],	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)

     # Test batched
     __A						      =			image_processing(_lowerCamelCase,	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)





@require_torch
@require_vision
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     Any									=			CLIPImageProcessor if is_vision_available() else None


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     __A						      =			CLIPImageProcessingTester(self,	num_channels=4  )
     __A						      =			3


   @property
   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     return self.image_processor_tester.prepare_image_processor_dict()


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_resize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''size'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_center_crop'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''center_crop'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_normalize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''image_mean'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''image_std'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,	'''do_convert_rgb'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     pass






   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     # Initialize image_processing
     __A						      =			self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A						      =			self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,	Image.Image  )

     # Test not batched input
     __A						      =			image_processing(image_inputs[0],	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             1,
             self.expected_encoded_image_num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)

     # Test batched
     __A						      =			image_processing(_lowerCamelCase,	return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,	(
             self.image_processor_tester.batch_size,
             self.expected_encoded_image_num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         ),	)












"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     str									=			ElectraTokenizer
   A_						:     List[Any]									=			ElectraTokenizerFast
   A_						:     Union[str, Any]									=			True
   A_						:     List[str]									=			True
   A_						:     Optional[Any]									=			filter_non_english


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     super().setUp()

     __A						      =			[
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     __A						      =			os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file,	'''w''',	encoding='''utf-8'''  ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple,	_lowerCamelCase			:					int  ):

     '''simple docstring'''
     __A						      =			'''UNwant\u00E9d,running'''
     __A						      =			'''unwanted, running'''
     return input_text, output_text


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class(self.vocab_file  )

     __A						      =			tokenizer.tokenize('''UNwant\u00E9d,running'''  )
     self.assertListEqual(_lowerCamelCase,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase  ),	[9, 6, 7, 12, 10, 11]  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     if not self.test_rust_tokenizer:
       return

     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     # With lower casing
     __A						      =			self.get_tokenizer(do_lower_case=_lowerCamelCase  )
     __A						      =			self.get_rust_tokenizer(do_lower_case=_lowerCamelCase  )

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  ),	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''h\u00E9llo''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	never_split=['''[UNK]''']  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

     __A						      =			{}
     for i, token in enumerate(_lowerCamelCase  ):
       __A						      =			i
     __A						      =			WordpieceTokenizer(vocab=_lowerCamelCase,	unk_token='''[UNK]'''  )

     self.assertListEqual(tokenizer.tokenize(''''''  ),	[]  )

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''  ),	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  ),	['''[UNK]''', '''runn''', '''##ing''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     self.assertTrue(_is_whitespace(''' '''  )  )
     self.assertTrue(_is_whitespace('''\t'''  )  )
     self.assertTrue(_is_whitespace('''\r'''  )  )
     self.assertTrue(_is_whitespace('''\n'''  )  )
     self.assertTrue(_is_whitespace('''\u00A0'''  )  )

     self.assertFalse(_is_whitespace('''A'''  )  )
     self.assertFalse(_is_whitespace('''-'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     self.assertTrue(_is_control('''\u0005'''  )  )

     self.assertFalse(_is_control('''A'''  )  )
     self.assertFalse(_is_control(''' '''  )  )
     self.assertFalse(_is_control('''\t'''  )  )
     self.assertFalse(_is_control('''\r'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     self.assertTrue(_is_punctuation('''-'''  )  )
     self.assertTrue(_is_punctuation('''$'''  )  )
     self.assertTrue(_is_punctuation('''`'''  )  )
     self.assertTrue(_is_punctuation('''.'''  )  )

     self.assertFalse(_is_punctuation('''A'''  )  )
     self.assertFalse(_is_punctuation(''' '''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )

     self.assertListEqual(
         [rust_tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''  )

     __A						      =			tokenizer.encode('''sequence builders''',	add_special_tokens=_lowerCamelCase  )
     __A						      =			tokenizer.encode('''multi-sequence build''',	add_special_tokens=_lowerCamelCase  )

     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  )
     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase,	_lowerCamelCase  )

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
         __A						      =			tokenizer_r.encode_plus(
             _lowerCamelCase,	return_attention_mask=_lowerCamelCase,	return_token_type_ids=_lowerCamelCase,	return_offsets_mapping=_lowerCamelCase,	add_special_tokens=_lowerCamelCase,	)

         __A						      =			tokenizer_r.do_lower_case if hasattr(_lowerCamelCase,	'''do_lower_case'''  ) else False
         __A						      =			(
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''A'''),
                 ((1, 2), ''','''),
                 ((3, 5), '''na'''),
                 ((5, 6), '''##ï'''),
                 ((6, 8), '''##ve'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''Allen'''),
                 ((21, 23), '''##NL'''),
                 ((23, 24), '''##P'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''a'''),
                 ((1, 2), ''','''),
                 ((3, 8), '''naive'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''allen'''),
                 ((21, 23), '''##nl'''),
                 ((23, 24), '''##p'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results],	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
         self.assertEqual([e[0] for e in expected_results],	tokens['''offset_mapping''']  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			['''的''', '''人''', '''有''']
     __A						      =			''''''.join(_lowerCamelCase  )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			True
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

         __A						      =			False
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that only the first Chinese character is not preceded by "##".
         __A						      =			[
             f'##{token}' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase  )
         ]
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )




