
import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 	A_       (      __lowerCamelCase				,     unittest.TestCase       ):




	'''simple docstring'''


	_UpperCamelCase					:					List[Any]										=	ElectraTokenizer
	_UpperCamelCase					:					List[Any]										=	ElectraTokenizerFast
	_UpperCamelCase					:					Tuple										=	True
	_UpperCamelCase					:					List[str]										=	True
	_UpperCamelCase					:					int										=	filter_non_english





	def      SCREAMING_SNAKE_CASE__						(	self	):
		super().setUp()

		lowercase					  =   [
		    '[UNK]',
		    '[CLS]',
		    '[SEP]',
		    '[PAD]',
		    '[MASK]',
		    'want',
		    '##want',
		    '##ed',
		    'wa',
		    'un',
		    'runn',
		    '##ing',
		    ',',
		    'low',
		    'lowest',
		]
		lowercase					  =   os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['vocab_file']	)
		with open(self.vocab_file		,   'w'		,   encoding='utf-8'	) as vocab_writer:
			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)





	def      SCREAMING_SNAKE_CASE__						(	self		,   snake_case	):
		lowercase					  =   'UNwant\u00E9d,running'
		lowercase					  =   'unwanted, running'
		return input_text, output_text





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   self.tokenizer_class(self.vocab_file	)

		lowercase					  =   tokenizer.tokenize('UNwant\u00E9d,running'	)
		self.assertListEqual(snake_case		,   ['un', '##want', '##ed', ',', 'runn', '##ing']	)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)		,   [9, 6, 7, 12, 10, 11]	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		if not self.test_rust_tokenizer:
			return

		lowercase					  =   self.get_tokenizer()
		lowercase					  =   self.get_rust_tokenizer()

		lowercase					  =   'UNwant\u00E9d,running'

		lowercase					  =   tokenizer.tokenize(snake_case	)
		lowercase					  =   rust_tokenizer.tokenize(snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)

		lowercase					  =   tokenizer.encode(snake_case		,   add_special_tokens=snake_case	)
		lowercase					  =   rust_tokenizer.encode(snake_case		,   add_special_tokens=snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)

		lowercase					  =   self.get_rust_tokenizer()
		lowercase					  =   tokenizer.encode(snake_case	)
		lowercase					  =   rust_tokenizer.encode(snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)

		# With lower casing
		lowercase					  =   self.get_tokenizer(do_lower_case=snake_case	)
		lowercase					  =   self.get_rust_tokenizer(do_lower_case=snake_case	)

		lowercase					  =   'UNwant\u00E9d,running'

		lowercase					  =   tokenizer.tokenize(snake_case	)
		lowercase					  =   rust_tokenizer.tokenize(snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)

		lowercase					  =   tokenizer.encode(snake_case		,   add_special_tokens=snake_case	)
		lowercase					  =   rust_tokenizer.encode(snake_case		,   add_special_tokens=snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)

		lowercase					  =   self.get_rust_tokenizer()
		lowercase					  =   tokenizer.encode(snake_case	)
		lowercase					  =   rust_tokenizer.encode(snake_case	)
		self.assertListEqual(snake_case		,   snake_case	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'	)		,   ['ah', '\u535A', '\u63A8', 'zz']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)		,   ['hello', '!', 'how', 'are', 'you', '?']	)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,   ['hello']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case		,   strip_accents=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,   ['hällo', '!', 'how', 'are', 'you', '?']	)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,   ['h\u00E9llo']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case		,   strip_accents=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,   ['hallo', '!', 'how', 'are', 'you', '?']	)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,   ['hello']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,   ['hallo', '!', 'how', 'are', 'you', '?']	)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,   ['hello']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)		,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case		,   strip_accents=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,   ['HäLLo', '!', 'how', 'Are', 'yoU', '?']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case		,   strip_accents=snake_case	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,   ['HaLLo', '!', 'how', 'Are', 'yoU', '?']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   BasicTokenizer(do_lower_case=snake_case		,   never_split=['[UNK]']	)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'	)		,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

		lowercase					  =   {}
		for i, token in enumerate(snake_case	):
			lowercase					  =   i
		lowercase					  =   WordpieceTokenizer(vocab=snake_case		,   unk_token='[UNK]'	)

		self.assertListEqual(tokenizer.tokenize(''	)		,   []	)

		self.assertListEqual(tokenizer.tokenize('unwanted running'	)		,   ['un', '##want', '##ed', 'runn', '##ing']	)

		self.assertListEqual(tokenizer.tokenize('unwantedX running'	)		,   ['[UNK]', 'runn', '##ing']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		self.assertTrue(_is_whitespace(' '	)	)
		self.assertTrue(_is_whitespace('\t'	)	)
		self.assertTrue(_is_whitespace('\r'	)	)
		self.assertTrue(_is_whitespace('\n'	)	)
		self.assertTrue(_is_whitespace('\u00A0'	)	)

		self.assertFalse(_is_whitespace('A'	)	)
		self.assertFalse(_is_whitespace('-'	)	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		self.assertTrue(_is_control('\u0005'	)	)

		self.assertFalse(_is_control('A'	)	)
		self.assertFalse(_is_control(' '	)	)
		self.assertFalse(_is_control('\t'	)	)
		self.assertFalse(_is_control('\r'	)	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		self.assertTrue(_is_punctuation('-'	)	)
		self.assertTrue(_is_punctuation('$'	)	)
		self.assertTrue(_is_punctuation('`'	)	)
		self.assertTrue(_is_punctuation('.'	)	)

		self.assertFalse(_is_punctuation('A'	)	)
		self.assertFalse(_is_punctuation(' '	)	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   self.get_tokenizer()
		lowercase					  =   self.get_rust_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(snake_case	) for t in ['Test', '\xad', 'test']]		,   [['[UNK]'], [], ['[UNK]']]	)

		self.assertListEqual(
		    [rust_tokenizer.tokenize(snake_case	) for t in ['Test', '\xad', 'test']]		,   [['[UNK]'], [], ['[UNK]']]	)





	@slow
	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   self.tokenizer_class.from_pretrained('google/electra-base-discriminator'	)

		lowercase					  =   tokenizer.encode('sequence builders'		,   add_special_tokens=snake_case	)
		lowercase					  =   tokenizer.encode('multi-sequence build'		,   add_special_tokens=snake_case	)

		lowercase					  =   tokenizer.build_inputs_with_special_tokens(snake_case	)
		lowercase					  =   tokenizer.build_inputs_with_special_tokens(snake_case		,   snake_case	)

		assert encoded_sentence == [101] + text + [102]
		assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      SCREAMING_SNAKE_CASE__						(	self	):
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
				lowercase					  =   self.rust_tokenizer_class.from_pretrained(snake_case		,   **snake_case	)

				lowercase					  =   F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
				lowercase					  =   tokenizer_r.encode_plus(
				    snake_case		,   return_attention_mask=snake_case		,   return_token_type_ids=snake_case		,   return_offsets_mapping=snake_case		,   add_special_tokens=snake_case		,   )

				lowercase					  =   tokenizer_r.do_lower_case if hasattr(snake_case		,   'do_lower_case'	) else False
				lowercase					  =   (
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'A'),
				        ((1, 2), ','),
				        ((3, 5), 'na'),
				        ((5, 6), '##ï'),
				        ((6, 8), '##ve'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'Allen'),
				        ((21, 23), '##NL'),
				        ((23, 24), '##P'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'a'),
				        ((1, 2), ','),
				        ((3, 8), 'naive'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'allen'),
				        ((21, 23), '##nl'),
				        ((23, 24), '##p'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results]		,   tokenizer_r.convert_ids_to_tokens(tokens['input_ids']	)	)
				self.assertEqual([e[0] for e in expected_results]		,   tokens['offset_mapping']	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   ['的', '人', '有']
		lowercase					  =   ''.join(snake_case	)
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
				lowercase					  =   True
				lowercase					  =   self.tokenizer_class.from_pretrained(snake_case		,   **snake_case	)
				lowercase					  =   self.rust_tokenizer_class.from_pretrained(snake_case		,   **snake_case	)

				lowercase					  =   tokenizer_p.encode(snake_case		,   add_special_tokens=snake_case	)
				lowercase					  =   tokenizer_r.encode(snake_case		,   add_special_tokens=snake_case	)

				lowercase					  =   tokenizer_r.convert_ids_to_tokens(snake_case	)
				lowercase					  =   tokenizer_p.convert_ids_to_tokens(snake_case	)

				# it is expected that each Chinese character is not preceded by "##"
				self.assertListEqual(snake_case		,   snake_case	)
				self.assertListEqual(snake_case		,   snake_case	)

				lowercase					  =   False
				lowercase					  =   self.rust_tokenizer_class.from_pretrained(snake_case		,   **snake_case	)
				lowercase					  =   self.tokenizer_class.from_pretrained(snake_case		,   **snake_case	)

				lowercase					  =   tokenizer_r.encode(snake_case		,   add_special_tokens=snake_case	)
				lowercase					  =   tokenizer_p.encode(snake_case		,   add_special_tokens=snake_case	)

				lowercase					  =   tokenizer_r.convert_ids_to_tokens(snake_case	)
				lowercase					  =   tokenizer_p.convert_ids_to_tokens(snake_case	)

				# it is expected that only the first Chinese character is not preceded by "##".
				lowercase					  =   [
				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(snake_case	)
				]
				self.assertListEqual(snake_case		,   snake_case	)
				self.assertListEqual(snake_case		,   snake_case	)



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
	import torch

if is_vision_available():
	from PIL import Image

	from transformers import VideoMAEImageProcessor







class 	A_       (      unittest.TestCase       ):




	'''simple docstring'''


	def __init__(	self		,   snake_case		,   snake_case=7		,   snake_case=3		,   snake_case=10		,   snake_case=18		,   snake_case=30		,   snake_case=400		,   snake_case=True		,   snake_case=None		,   snake_case=True		,   snake_case=[0.5, 0.5, 0.5]		,   snake_case=[0.5, 0.5, 0.5]		,   snake_case=None		,   ):
		lowercase					  =   size if size is not None else {'shortest_edge': 18}
		lowercase					  =   crop_size if crop_size is not None else {'height': 18, 'width': 18}

		lowercase					  =   parent
		lowercase					  =   batch_size
		lowercase					  =   num_channels
		lowercase					  =   num_frames
		lowercase					  =   image_size
		lowercase					  =   min_resolution
		lowercase					  =   max_resolution
		lowercase					  =   do_resize
		lowercase					  =   size
		lowercase					  =   do_normalize
		lowercase					  =   image_mean
		lowercase					  =   image_std
		lowercase					  =   crop_size





	def      SCREAMING_SNAKE_CASE__						(	self	):
		return {
		    "image_mean": self.image_mean,
		    "image_std": self.image_std,
		    "do_normalize": self.do_normalize,
		    "do_resize": self.do_resize,
		    "size": self.size,
		    "crop_size": self.crop_size,
		}







@require_torch
@require_vision
class 	A_       (      __lowerCamelCase				,     unittest.TestCase       ):




	'''simple docstring'''


	_UpperCamelCase					:					List[str]										=	VideoMAEImageProcessor if is_vision_available() else None





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   VideoMAEImageProcessingTester(self	)





	@property
	def      SCREAMING_SNAKE_CASE__						(	self	):
		return self.image_processor_tester.prepare_image_processor_dict()





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   self.image_processing_class(**self.image_processor_dict	)
		self.assertTrue(hasattr(snake_case		,   'image_mean'	)	)
		self.assertTrue(hasattr(snake_case		,   'image_std'	)	)
		self.assertTrue(hasattr(snake_case		,   'do_normalize'	)	)
		self.assertTrue(hasattr(snake_case		,   'do_resize'	)	)
		self.assertTrue(hasattr(snake_case		,   'do_center_crop'	)	)
		self.assertTrue(hasattr(snake_case		,   'size'	)	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		lowercase					  =   self.image_processing_class.from_dict(self.image_processor_dict	)
		self.assertEqual(image_processor.size		,   {'shortest_edge': 18}	)
		self.assertEqual(image_processor.crop_size		,   {'height': 18, 'width': 18}	)

		lowercase					  =   self.image_processing_class.from_dict(self.image_processor_dict		,   size=42		,   crop_size=84	)
		self.assertEqual(image_processor.size		,   {'shortest_edge': 42}	)
		self.assertEqual(image_processor.crop_size		,   {'height': 84, 'width': 84}	)





	def      SCREAMING_SNAKE_CASE__						(	self	):
		pass





	def      SCREAMING_SNAKE_CASE__						(	self	):
		# Initialize image_processing
		lowercase					  =   self.image_processing_class(**self.image_processor_dict	)
		# create random PIL videos
		lowercase					  =   prepare_video_inputs(self.image_processor_tester		,   equal_resolution=snake_case	)
		for video in video_inputs:
			self.assertIsInstance(snake_case		,   snake_case	)
			self.assertIsInstance(video[0]		,   Image.Image	)

		# Test not batched input
		lowercase					  =   image_processing(video_inputs[0]		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        1,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )

		# Test batched
		lowercase					  =   image_processing(snake_case		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        self.image_processor_tester.batch_size,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )





	def      SCREAMING_SNAKE_CASE__						(	self	):
		# Initialize image_processing
		lowercase					  =   self.image_processing_class(**self.image_processor_dict	)
		# create random numpy tensors
		lowercase					  =   prepare_video_inputs(self.image_processor_tester		,   equal_resolution=snake_case		,   numpify=snake_case	)
		for video in video_inputs:
			self.assertIsInstance(snake_case		,   snake_case	)
			self.assertIsInstance(video[0]		,   np.ndarray	)

		# Test not batched input
		lowercase					  =   image_processing(video_inputs[0]		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        1,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )

		# Test batched
		lowercase					  =   image_processing(snake_case		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        self.image_processor_tester.batch_size,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )





	def      SCREAMING_SNAKE_CASE__						(	self	):
		# Initialize image_processing
		lowercase					  =   self.image_processing_class(**self.image_processor_dict	)
		# create random PyTorch tensors
		lowercase					  =   prepare_video_inputs(self.image_processor_tester		,   equal_resolution=snake_case		,   torchify=snake_case	)
		for video in video_inputs:
			self.assertIsInstance(snake_case		,   snake_case	)
			self.assertIsInstance(video[0]		,   torch.Tensor	)

		# Test not batched input
		lowercase					  =   image_processing(video_inputs[0]		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        1,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )

		# Test batched
		lowercase					  =   image_processing(snake_case		,   return_tensors='pt'	).pixel_values
		self.assertEqual(
		    encoded_videos.shape		,   (
		        self.image_processor_tester.batch_size,
		        self.image_processor_tester.num_frames,
		        self.image_processor_tester.num_channels,
		        self.image_processor_tester.crop_size['height'],
		        self.image_processor_tester.crop_size['width'],
		    )		,   )

