
'''simple docstring'''





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import VideoMAEImageProcessor


class 	UpperCamelCase__					(      unittest.TestCase):






 def __init__(     self					:Tuple		,			_A					:Union[str, Any]		,			_A					:int=7		,			_A					:Dict=3		,			_A					:Dict=10		,			_A					:Optional[int]=18		,			_A					:Dict=30		,			_A					:str=400		,			_A					:Optional[Any]=True		,			_A					:Dict=None		,			_A					:Optional[Any]=True		,			_A					:int=[0.5, 0.5, 0.5]		,			_A					:str=[0.5, 0.5, 0.5]		,			_A					:Tuple=None		,			)			->     Tuple:







       '''simple docstring'''



       __A    		=     size if size is not None else {'shortest_edge': 18}
       __A    		=     crop_size if crop_size is not None else {'height': 18, 'width': 18}

       __A    		=     parent
       __A    		=     batch_size
       __A    		=     num_channels
       __A    		=     num_frames
       __A    		=     image_size
       __A    		=     min_resolution
       __A    		=     max_resolution
       __A    		=     do_resize
       __A    		=     size
       __A    		=     do_normalize
       __A    		=     image_mean
       __A    		=     image_std
       __A    		=     crop_size






 def 		lowercase_  (     self					:int     )			->     Union[str, Any]:







       '''simple docstring'''



       return {
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_normalize": self.do_normalize,
           "do_resize": self.do_resize,
           "size": self.size,
           "crop_size": self.crop_size,
       }


@require_torch
@require_vision
class 	UpperCamelCase__					(      SCREAMING_SNAKE_CASE ,       unittest.TestCase):
 UpperCAmelCase__      :      str  			=   VideoMAEImageProcessor if is_vision_available() else None






 def 		lowercase_  (     self					:Tuple     )			->     Union[str, Any]:







       '''simple docstring'''



       __A    		=     VideoMAEImageProcessingTester(self     )






 @property
 def 		lowercase_  (     self					:Dict     )			->     int:







       '''simple docstring'''



       return self.image_processor_tester.prepare_image_processor_dict()






 def 		lowercase_  (     self					:Optional[int]     )			->     Optional[Any]:







       '''simple docstring'''



       __A    		=     self.image_processing_class(**self.image_processor_dict     )
       self.assertTrue(hasattr(_A		,			'image_mean'     )     )
       self.assertTrue(hasattr(_A		,			'image_std'     )     )
       self.assertTrue(hasattr(_A		,			'do_normalize'     )     )
       self.assertTrue(hasattr(_A		,			'do_resize'     )     )
       self.assertTrue(hasattr(_A		,			'do_center_crop'     )     )
       self.assertTrue(hasattr(_A		,			'size'     )     )






 def 		lowercase_  (     self					:List[str]     )			->     List[str]:







       '''simple docstring'''



       __A    		=     self.image_processing_class.from_dict(self.image_processor_dict     )
       self.assertEqual(image_processor.size		,			{'shortest_edge': 18}     )
       self.assertEqual(image_processor.crop_size		,			{'height': 18, 'width': 18}     )

       __A    		=     self.image_processing_class.from_dict(self.image_processor_dict		,			size=42		,			crop_size=84     )
       self.assertEqual(image_processor.size		,			{'shortest_edge': 42}     )
       self.assertEqual(image_processor.crop_size		,			{'height': 84, 'width': 84}     )






 def 		lowercase_  (     self					:int     )			->     int:







       '''simple docstring'''



       pass






 def 		lowercase_  (     self					:List[str]     )			->     List[str]:







       '''simple docstring'''



       __A    		=     self.image_processing_class(**self.image_processor_dict     )
       # create random PIL videos
       __A    		=     prepare_video_inputs(self.image_processor_tester		,			equal_resolution=_A     )
       for video in video_inputs:
             self.assertIsInstance(_A		,			_A     )
             self.assertIsInstance(video[0]		,			Image.Image     )

       # Test not batched input
       __A    		=     image_processing(video_inputs[0]		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)

       # Test batched
       __A    		=     image_processing(_A		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)






 def 		lowercase_  (     self					:int     )			->     Dict:







       '''simple docstring'''



       __A    		=     self.image_processing_class(**self.image_processor_dict     )
       # create random numpy tensors
       __A    		=     prepare_video_inputs(self.image_processor_tester		,			equal_resolution=_A		,			numpify=_A     )
       for video in video_inputs:
             self.assertIsInstance(_A		,			_A     )
             self.assertIsInstance(video[0]		,			np.ndarray     )

       # Test not batched input
       __A    		=     image_processing(video_inputs[0]		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)

       # Test batched
       __A    		=     image_processing(_A		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)






 def 		lowercase_  (     self					:Any     )			->     List[str]:







       '''simple docstring'''



       __A    		=     self.image_processing_class(**self.image_processor_dict     )
       # create random PyTorch tensors
       __A    		=     prepare_video_inputs(self.image_processor_tester		,			equal_resolution=_A		,			torchify=_A     )
       for video in video_inputs:
             self.assertIsInstance(_A		,			_A     )
             self.assertIsInstance(video[0]		,			torch.Tensor     )

       # Test not batched input
       __A    		=     image_processing(video_inputs[0]		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)

       # Test batched
       __A    		=     image_processing(_A		,			return_tensors='pt'     ).pixel_values
       self.assertEqual(
           encoded_videos.shape		,			(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )		,			)








'''simple docstring'''





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	UpperCamelCase__					(      SCREAMING_SNAKE_CASE ,       unittest.TestCase):
 UpperCAmelCase__      :      Tuple  			=   PreTrainedTokenizerFast
 UpperCAmelCase__      :      Tuple  			=   False
 UpperCAmelCase__      :      str  			=   True
 UpperCAmelCase__      :      str  			=   'tokenizer_file'






 def 		lowercase_  (     self					:Union[str, Any]     )			->     List[Any]:







       '''simple docstring'''



       __A    		=     False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       __A    		=     True

       __A    		=     ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
       __A    		=     'SaulLu/dummy-tokenizer-bytelevel-bpe'

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       __A    		=     [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       __A    		=     PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
       tokenizer.save_pretrained(self.tmpdirname     )






 def 		lowercase_  (     self					:Any     )			->     List[str]:







       '''simple docstring'''



       pass






 def 		lowercase_  (     self					:Optional[Any]     )			->     Optional[Any]:







       '''simple docstring'''



       pass






 def 		lowercase_  (     self					:Tuple     )			->     str:







       '''simple docstring'''



       pass






 def 		lowercase_  (     self					:List[Any]     )			->     Tuple:







       '''simple docstring'''



       pass






 def 		lowercase_  (     self					:List[Any]     )			->     str:







       '''simple docstring'''



       __A    		=     self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                   try:
                         __A    		=     tempfile.mkdtemp()
                         __A    		=     self.rust_tokenizer_class.from_pretrained(_A		,			**_A     )

                         tokenizer.save_pretrained(self.tmpdirname     )
                         super().test_training_new_tokenizer()
                   finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname     )
                         __A    		=     tmpdirname_orig






 def 		lowercase_  (     self					:int     )			->     int:







       '''simple docstring'''



       __A    		=     self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                   try:
                         __A    		=     tempfile.mkdtemp()
                         __A    		=     self.rust_tokenizer_class.from_pretrained(_A		,			**_A     )

                         tokenizer.save_pretrained(self.tmpdirname     )
                         super().test_training_new_tokenizer_with_special_tokens_change()
                   finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname     )
                         __A    		=     tmpdirname_orig






 def 		lowercase_  (     self					:Optional[Any]     )			->     Any:







       '''simple docstring'''



       __A    		=     self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

       __A    		=     ('a' for _ in range(1_000     ))
       __A    		=     tokenizer.train_new_from_iterator(text_iterator=_A		,			length=1_000		,			vocab_size=50     )

       __A    		=     new_tokenizer.encode('a🤗'     )
       self.assertEqual(_A		,			[64, 172, 253, 97, 245]     )


@require_tokenizers
class 	UpperCamelCase__					(      unittest.TestCase):






 def 		lowercase_  (     self					:Tuple     )			->     List[Any]:







       '''simple docstring'''



       __A    		=     AutoTokenizer.from_pretrained('bert-base-cased'     )
       __A    		=     json.loads(tokenizer._tokenizer.to_str()     )
       __A    		=     len(_A     )

       with tempfile.TemporaryDirectory() as tmp_dir:
             # Hack to save this in the tokenizer_config.json
             __A    		=     ['tokenizer.4.0.0.json']
             tokenizer.save_pretrained(_A     )
             json.dump(_A		,			open(os.path.join(_A		,			'tokenizer.4.0.0.json'     )		,			'w'     )     )

             # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
             __A    		=     AutoTokenizer.from_pretrained(_A     )
             self.assertEqual(len(_A     )		,			len(_A     ) + 1     )
             __A    		=     json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertIn('huggingface'		,			json_tokenizer['model']['vocab']     )

             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(_A		,			'tokenizer.4.0.0.json'     )		,			os.path.join(_A		,			'tokenizer.42.0.0.json'     )     )
             __A    		=     ['tokenizer.42.0.0.json']
             tokenizer.save_pretrained(_A     )
             __A    		=     AutoTokenizer.from_pretrained(_A     )
             self.assertEqual(len(_A     )		,			len(_A     )     )
             __A    		=     json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertNotIn('huggingface'		,			json_tokenizer['model']['vocab']     )






 def 		lowercase_  (     self					:List[Any]     )			->     Optional[int]:







       '''simple docstring'''



       __A    		=     'hf-internal-testing/test-two-tokenizers'

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       __A    		=     AutoTokenizer.from_pretrained(_A     )
       self.assertEqual(len(_A     )		,			28_997     )
       __A    		=     json.loads(tokenizer._tokenizer.to_str()     )
       self.assertIn('huggingface'		,			json_tokenizer['model']['vocab']     )

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       __A    		=     '3.0.0'
       __A    		=     old_transformers.models.auto.AutoTokenizer.from_pretrained(_A     )
       self.assertEqual(len(_A     )		,			28_996     )
       __A    		=     json.loads(old_tokenizer._tokenizer.to_str()     )
       self.assertNotIn('huggingface'		,			json_tokenizer['model']['vocab']     )


@require_tokenizers
class 	UpperCamelCase__					(      unittest.TestCase):






 def 		lowercase_  (     self					:List[str]     )			->     List[Any]:







       '''simple docstring'''



       __A    		=     PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'     )
       __A    		=     'The Matrix is a 1999 science fiction action film.'

       with concurrent.futures.ThreadPoolExecutor() as executor:
             __A    		=     [executor.submit(self.fetch		,			_A		,			_A     ) for i in range(10     )]
             __A    		=     [future.result() for future in futures]
             self.assertEqual(_A		,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10     )]     )






 def 		lowercase_  (     self					:Optional[int]		,			_A					:str		,			_A					:Any     )			->     Optional[int]:







       '''simple docstring'''



       return tokenizer.encode(_A		,			truncation='longest_first'		,			padding='longest'     )






