






import json
import os
import tempfile

from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
  import numpy as np
  import torch

if is_vision_available():
  from PIL import Image



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[str]     ,			__lowercase				:     str=False     ,			__lowercase				:     List[str]=False     ,			__lowercase				:     Dict=False      )  -> Union[str, Any]:





    """simple docstring"""

    assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

    __A             =		[]
    for i in range(image_processor_tester.batch_size      ):
        if equal_resolution:
            __A             =		__A             =		image_processor_tester.max_resolution
        else:
            # To avoid getting image width/height 0
            __A             =		image_processor_tester.min_resolution
            if getattr(__lowercase     ,			"""size_divisor"""     ,			__lowercase      ):
                # If `size_divisor` is defined, the image needs to have width/size >= `size_divisor`
                __A             =		max(image_processor_tester.size_divisor     ,			__lowercase      )
            __A      ,		__A             =		np.random.choice(np.arange(__lowercase     ,			image_processor_tester.max_resolution      )     ,			2      )
        image_inputs.append(
            np.random.randint(2_5_5     ,			size=(image_processor_tester.num_channels, width, height)     ,			dtype=np.uinta      )      )

    if not numpify and not torchify:
        # PIL expects the channel dimension as last dimension
        __A             =		[Image.fromarray(np.moveaxis(__lowercase     ,			0     ,			-1      )      ) for image in image_inputs]

    if torchify:
        __A             =		[torch.from_numpy(__lowercase      ) for image in image_inputs]

    return image_inputs



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[Any]     ,			__lowercase				:     Union[str, Any]=1_0     ,			__lowercase				:     int=1_0     ,			__lowercase				:     Dict=False     ,			__lowercase				:     Optional[Any]=False      )  -> List[str]:





    """simple docstring"""

    __A             =		[]
    for i in range(image_processor_tester.num_frames      ):
        video.append(np.random.randint(2_5_5     ,			size=(image_processor_tester.num_channels, width, height)     ,			dtype=np.uinta      )      )

    if not numpify and not torchify:
        # PIL expects the channel dimension as last dimension
        __A             =		[Image.fromarray(np.moveaxis(__lowercase     ,			0     ,			-1      )      ) for frame in video]

    if torchify:
        __A             =		[torch.from_numpy(__lowercase      ) for frame in video]

    return video



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[str]     ,			__lowercase				:     Any=False     ,			__lowercase				:     Union[str, Any]=False     ,			__lowercase				:     Tuple=False      )  -> List[Any]:





    """simple docstring"""

    assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

    __A             =		[]
    for i in range(image_processor_tester.batch_size      ):
        if equal_resolution:
            __A             =		__A             =		image_processor_tester.max_resolution
        else:
            __A      ,		__A             =		np.random.choice(
                np.arange(image_processor_tester.min_resolution     ,			image_processor_tester.max_resolution      )     ,			2      )
            __A             =		prepare_video(
                image_processor_tester=__lowercase     ,			width=__lowercase     ,			height=__lowercase     ,			numpify=__lowercase     ,			torchify=__lowercase     ,			)
        video_inputs.append(__lowercase      )

    return video_inputs






class  __lowercase :







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			None




     def 						lowerCAmelCase_       (			self    :     List[Any]			):


         """simple docstring"""



         __A             =		self.image_processing_class(**self.image_processor_dict			)
         __A             =		json.loads(image_processor.to_json_string()			)
         for key, value in self.image_processor_dict.items():
             self.assertEqual(obj[key]	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		self.image_processing_class(**self.image_processor_dict			)

         with tempfile.TemporaryDirectory() as tmpdirname:
             __A             =		os.path.join(UpperCamelCase_	,		"""image_processor.json"""			)
             image_processor_first.to_json_file(UpperCamelCase_			)
             __A             =		self.image_processing_class.from_json_file(UpperCamelCase_			)

         self.assertEqual(image_processor_second.to_dict()	,		image_processor_first.to_dict()			)




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		self.image_processing_class(**self.image_processor_dict			)

         with tempfile.TemporaryDirectory() as tmpdirname:
             __A             =		image_processor_first.save_pretrained(UpperCamelCase_			)[0]
             check_json_file_has_correct_format(UpperCamelCase_			)
             __A             =		self.image_processing_class.from_pretrained(UpperCamelCase_			)

         self.assertEqual(image_processor_second.to_dict()	,		image_processor_first.to_dict()			)




     def 						lowerCAmelCase_       (			self    :     int			):


         """simple docstring"""



         __A             =		self.image_processing_class()
         self.assertIsNotNone(UpperCamelCase_			)





     @require_torch
     @require_vision
     def 						lowerCAmelCase_       (			self    :     List[Any]			):


         """simple docstring"""



         if self.test_cast_dtype is not None:
             # Initialize image_processor
             __A             =		self.image_processing_class(**self.image_processor_dict			)

             # create random PyTorch tensors
             __A             =		prepare_image_inputs(self.image_processor_tester	,		equal_resolution=UpperCamelCase_	,		torchify=UpperCamelCase_			)

             __A             =		image_processor(UpperCamelCase_	,		return_tensors="""pt"""			)
             # for layoutLM compatiblity
             self.assertEqual(encoding.pixel_values.device	,		torch.device("""cpu"""			)			)
             self.assertEqual(encoding.pixel_values.dtype	,		torch.floataa			)

             __A             =		image_processor(UpperCamelCase_	,		return_tensors="""pt"""			).to(torch.floataa			)
             self.assertEqual(encoding.pixel_values.device	,		torch.device("""cpu"""			)			)
             self.assertEqual(encoding.pixel_values.dtype	,		torch.floataa			)

             __A             =		image_processor(UpperCamelCase_	,		return_tensors="""pt"""			).to("""cpu"""	,		torch.bfloataa			)
             self.assertEqual(encoding.pixel_values.device	,		torch.device("""cpu"""			)			)
             self.assertEqual(encoding.pixel_values.dtype	,		torch.bfloataa			)

             with self.assertRaises(UpperCamelCase_			):
                 __A             =		image_processor(UpperCamelCase_	,		return_tensors="""pt"""			).to(torch.bfloataa	,		"""cpu"""			)

             # Try with text + image feature
             __A             =		image_processor(UpperCamelCase_	,		return_tensors="""pt"""			)
             encoding.update({"""input_ids""": torch.LongTensor([[1, 2, 3], [4, 5, 6]]			)}			)
             __A             =		encoding.to(torch.floataa			)

             self.assertEqual(encoding.pixel_values.device	,		torch.device("""cpu"""			)			)
             self.assertEqual(encoding.pixel_values.dtype	,		torch.floataa			)
             self.assertEqual(encoding.input_ids.dtype	,		torch.long			)












import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__a :    Optional[Any]								=				get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class  __lowercase (		lowercase_     , unittest.TestCase					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			ErnieMTokenizer
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			True
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			False




     def 						lowerCAmelCase_       (			self    :     Optional[Any]			):


         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)
         tokenizer.save_pretrained(self.tmpdirname			)




     def 						lowerCAmelCase_       (			self    :     Optional[Any]	,		UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         __A             =		"""this is a test"""
         __A             =		"""this is a test"""
         return input_text, output_text




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		"""<pad>"""
         __A             =		0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_			)	,		UpperCamelCase_			)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_			)	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         __A             =		list(self.get_tokenizer().get_vocab().keys()			)

         self.assertEqual(vocab_keys[0]	,		"""<pad>"""			)
         self.assertEqual(vocab_keys[1]	,		"""<unk>"""			)
         self.assertEqual(vocab_keys[-1]	,		"""▁eloquent"""			)
         self.assertEqual(len(UpperCamelCase_			)	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         if not self.test_rust_tokenizer:
             return

         __A             =		self.get_tokenizer()
         __A             =		self.get_rust_tokenizer()

         __A             =		"""I was born in 92000, and this is falsé."""

         __A             =		tokenizer.tokenize(UpperCamelCase_			)
         __A             =		rust_tokenizer.tokenize(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		self.get_rust_tokenizer()
         __A             =		tokenizer.encode(UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		do_lower_case=UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.tokenize("""This is a test"""			)
         self.assertListEqual(UpperCamelCase_	,		["""▁this""", """▁is""", """▁a""", """▁test"""]			)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_			)	,		[48, 25, 21, 1_289]			)

         __A             =		tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
         # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]			)
         __A             =		tokenizer.convert_tokens_to_ids(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		[31, 23, 386, 19, 518, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]			)

         __A             =		tokenizer.convert_ids_to_tokens(UpperCamelCase_			)
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]	,		)




     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.encode("""sequence builders"""			)
         __A             =		tokenizer.encode("""multi-sequence build"""			)

         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			)
         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_	,		UpperCamelCase_			)

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
             tokenizer.sep_token_id
         ] + text_a + [tokenizer.sep_token_id]





     @slow
     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		{"""input_ids""": [[0, 11_062, 82_772, 7, 15, 82_772, 538, 51_529, 237, 17_198, 1_290, 206, 9, 215_175, 1_314, 136, 17_198, 1_290, 206, 9, 56_359, 42, 122_009, 9, 16_466, 16, 87_344, 4_537, 9, 4_717, 78_381, 6, 159_958, 7, 15, 24_480, 618, 4, 527, 22_693, 9, 304, 4, 2_777, 24_480, 9_874, 4, 43_523, 594, 4, 803, 18_392, 33_189, 18, 4, 43_523, 24_447, 5, 5, 5, 16, 100, 24_955, 83_658, 9_626, 144_057, 15, 839, 22_335, 16, 136, 24_955, 83_658, 83_479, 15, 39_102, 724, 16, 678, 645, 6_460, 1_328, 4_589, 42, 122_009, 115_774, 23, 3_559, 1_328, 46_876, 7, 136, 53_894, 1_940, 42_227, 41_159, 17_721, 823, 425, 4, 27_512, 98_722, 206, 136, 5_531, 4_970, 919, 17_336, 5, 2], [0, 20_080, 618, 83, 82_775, 47, 479, 9, 1_517, 73, 53_894, 333, 80_581, 110_117, 18_811, 5_256, 1_295, 51, 152_526, 297, 7_986, 390, 124_416, 538, 35_431, 214, 98, 15_044, 25_737, 136, 7_108, 43_701, 23, 756, 135_355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63_773, 119_455, 6, 147_797, 88_203, 7, 645, 70, 21, 3_285, 10_269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=UpperCamelCase_	,		model_name="""susnato/ernie-m-base_pytorch"""	,		sequences=[
                 """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
                 """general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
                 """Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained """
                 """models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
                 """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
                 """conditioning on both left and right context in all layers.""",
                 """The quick brown fox jumps over the lazy dog.""",
             ]	,		)




