





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFXLMForMultipleChoice,
				    TFXLMForQuestionAnsweringSimple,
				    TFXLMForSequenceClassification,
				    TFXLMForTokenClassification,
				    TFXLMModel,
				    TFXLMWithLMHeadModel,
				    XLMConfig,
				)





class 						_A:


				"""simple docstring"""


				def __init__(      self					, _A					, ):
									__A     :						List[Any]          =    parent
									__A     :						Any          =    13
									__A     :						Dict          =    7
									__A     :						Dict          =    True
									__A     :						Union[str, Any]          =    True
									__A     :						str          =    True
									__A     :						Union[str, Any]          =    True
									__A     :						str          =    True
									__A     :						Optional[Any]          =    False
									__A     :						Optional[int]          =    False
									__A     :						str          =    False
									__A     :						str          =    2
									__A     :						Tuple          =    99
									__A     :						Optional[Any]          =    0
									__A     :						Union[str, Any]          =    32
									__A     :						List[Any]          =    2
									__A     :						str          =    4
									__A     :						int          =    0.1
									__A     :						str          =    0.1
									__A     :						Optional[Any]          =    512
									__A     :						int          =    16
									__A     :						Any          =    2
									__A     :						Dict          =    0.0_2
									__A     :						int          =    3
									__A     :						str          =    4
									__A     :						Union[str, Any]          =    'last'
									__A     :						Dict          =    True
									__A     :						Dict          =    None
									__A     :						List[str]          =    0



				def  UpperCAmelCase_		(      self					):
									__A     :						Any          =    ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size					)
									__A     :						Optional[int]          =    random_attention_mask([self.batch_size, self.seq_length]					, dtype=tf.floataa					)

									__A     :						str          =    None
									if self.use_input_lengths:
														__A     :						Union[str, Any]          =    (
														    ids_tensor([self.batch_size]					, vocab_size=2					) + self.seq_length - 2
														)  # small variation of seq_length

									__A     :						int          =    None
									if self.use_token_type_ids:
														__A     :						int          =    ids_tensor([self.batch_size, self.seq_length]					, self.n_langs					)

									__A     :						Optional[int]          =    None
									__A     :						Union[str, Any]          =    None
									__A     :						Optional[int]          =    None
									if self.use_labels:
														__A     :						Optional[Any]          =    ids_tensor([self.batch_size]					, self.type_sequence_label_size					)
														__A     :						Tuple          =    ids_tensor([self.batch_size, self.seq_length]					, self.num_labels					)
														__A     :						Optional[Any]          =    ids_tensor([self.batch_size]					, 2					, dtype=tf.floataa					)
														__A     :						List[Any]          =    ids_tensor([self.batch_size]					, self.num_choices					)

									__A     :						Optional[Any]          =    XLMConfig(
									    vocab_size=self.vocab_size					, n_special=self.n_special					, emb_dim=self.hidden_size					, n_layers=self.num_hidden_layers					, n_heads=self.num_attention_heads					, dropout=self.hidden_dropout_prob					, attention_dropout=self.attention_probs_dropout_prob					, gelu_activation=self.gelu_activation					, sinusoidal_embeddings=self.sinusoidal_embeddings					, asm=self.asm					, causal=self.causal					, n_langs=self.n_langs					, max_position_embeddings=self.max_position_embeddings					, initializer_range=self.initializer_range					, summary_type=self.summary_type					, use_proj=self.use_proj					, bos_token_id=self.bos_token_id					, )

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Tuple          =    TFXLMModel(config=_A					)
									__A     :						Optional[int]          =    {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
									__A     :						Any          =    model(_A					)

									__A     :						int          =    [input_ids, input_mask]
									__A     :						Optional[Any]          =    model(_A					)
									self.parent.assertEqual(result.last_hidden_state.shape					, (self.batch_size, self.seq_length, self.hidden_size)					)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Any          =    TFXLMWithLMHeadModel(_A					)

									__A     :						Dict          =    {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
									__A     :						str          =    model(_A					)

									__A     :						str          =    outputs

									self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.vocab_size)					)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Tuple          =    TFXLMForQuestionAnsweringSimple(_A					)

									__A     :						Tuple          =    {'input_ids': input_ids, 'lengths': input_lengths}

									__A     :						Dict          =    model(_A					)

									self.parent.assertEqual(result.start_logits.shape					, (self.batch_size, self.seq_length)					)
									self.parent.assertEqual(result.end_logits.shape					, (self.batch_size, self.seq_length)					)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Optional[Any]          =    TFXLMForSequenceClassification(_A					)

									__A     :						Dict          =    {'input_ids': input_ids, 'lengths': input_lengths}

									__A     :						int          =    model(_A					)

									self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.type_sequence_label_size)					)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Any          =    self.num_labels
									__A     :						Optional[Any]          =    TFXLMForTokenClassification(config=_A					)
									__A     :						List[Any]          =    {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
									__A     :						Optional[int]          =    model(_A					)
									self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.num_labels)					)



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, _A					, ):
									__A     :						Any          =    self.num_choices
									__A     :						Dict          =    TFXLMForMultipleChoice(config=_A					)
									__A     :						str          =    tf.tile(tf.expand_dims(_A					, 1					)					, (1, self.num_choices, 1)					)
									__A     :						Union[str, Any]          =    tf.tile(tf.expand_dims(_A					, 1					)					, (1, self.num_choices, 1)					)
									__A     :						Optional[Any]          =    tf.tile(tf.expand_dims(_A					, 1					)					, (1, self.num_choices, 1)					)
									__A     :						Optional[int]          =    {
									    'input_ids': multiple_choice_inputs_ids,
									    'attention_mask': multiple_choice_input_mask,
									    'token_type_ids': multiple_choice_token_type_ids,
									}
									__A     :						Dict          =    model(_A					)
									self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.num_choices)					)





				def  UpperCAmelCase_		(      self					):
									__A     :						Dict          =    self.prepare_config_and_inputs()
									(
									    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    (
									    __A
									)   ,    
									)     :						List[str]          =    config_and_inputs
									__A     :						Tuple          =    {
									    'input_ids': input_ids,
									    'token_type_ids': token_type_ids,
									    'langs': token_type_ids,
									    'lengths': input_lengths,
									}
									return config, inputs_dict





@require_tf
class 						_A( snake_case__    ,			snake_case__    ,			unittest.TestCase       ):


				"""simple docstring"""


				UpperCamelCase   :  Optional[int]    =     (
				    (
				        TFXLMModel,
				        TFXLMWithLMHeadModel,
				        TFXLMForSequenceClassification,
				        TFXLMForQuestionAnsweringSimple,
				        TFXLMForTokenClassification,
				        TFXLMForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCamelCase   :  str    =     (
				    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
				)  # TODO (PVP): Check other models whether language generation is also applicable
				UpperCamelCase   :  str    =     (
				    {
				        '''feature-extraction''': TFXLMModel,
				        '''fill-mask''': TFXLMWithLMHeadModel,
				        '''question-answering''': TFXLMForQuestionAnsweringSimple,
				        '''text-classification''': TFXLMForSequenceClassification,
				        '''text-generation''': TFXLMWithLMHeadModel,
				        '''token-classification''': TFXLMForTokenClassification,
				        '''zero-shot''': TFXLMForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				UpperCamelCase   :  Union[str, Any]    =     False
				UpperCamelCase   :  Tuple    =     False



				def  UpperCAmelCase_		(      self					, _A					, _A					, _A					, _A					, _A					):
									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith('Fast'					)
									):
														# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
														# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
														# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
														return True

									return False



				def  UpperCAmelCase_		(      self					):
									__A     :						Dict          =    TFXLMModelTester(self					)
									__A     :						str          =    ConfigTester(self					, config_class=_A					, emb_dim=37					)



				def  UpperCAmelCase_		(      self					):
									self.config_tester.run_common_tests()



				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_model(*_A					)



				def  UpperCAmelCase_		(      self					):
									__A     :						Tuple          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_lm_head(*_A					)



				def  UpperCAmelCase_		(      self					):
									__A     :						Optional[Any]          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_qa(*_A					)



				def  UpperCAmelCase_		(      self					):
									__A     :						Dict          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_sequence_classif(*_A					)



				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_for_token_classification(*_A					)



				def  UpperCAmelCase_		(      self					):
									__A     :						Optional[int]          =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_xlm_for_multiple_choice(*_A					)





				@slow
				def  UpperCAmelCase_		(      self					):
									for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														__A     :						Tuple          =    TFXLMModel.from_pretrained(_A					)
														self.assertIsNotNone(_A					)





@require_tf
class 						_A( unittest.TestCase       ):


				"""simple docstring"""


				@slow
				def  UpperCAmelCase_		(      self					):
									__A     :						Tuple          =    TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'					)
									__A     :						Dict          =    tf.convert_to_tensor([[14, 447]]					, dtype=tf.intaa					)  # the president
									__A     :						int          =    [
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									    14,
									    447,
									]  # the president the president the president the president the president the president the president the president the president the president
									# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
									__A     :						str          =    model.generate(_A					, do_sample=_A					)
									self.assertListEqual(output_ids[0].numpy().tolist()					, _A					)











import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import TvltImageProcessor







def 			_SCREAMING_SNAKE_CASE    (       a ,						a=10 ,						a=10 ,						a=False ,						a=False  )				->				Union[str, Any]:
					__A     :						str          =    []
					for i in range(image_processor_tester.num_frames  ):
										video.append(np.random.randint(2_55 ,						size=(image_processor_tester.num_channels, width, height) ,						dtype=np.uinta  )  )

					if not numpify and not torchify:
										# PIL expects the channel dimension as last dimension
										__A     :						Union[str, Any]          =    [Image.fromarray(np.moveaxis(a ,						0 ,						-1  )  ) for frame in video]

					if torchify:
										__A     :						Dict          =    [torch.from_numpy(a  ) for frame in video]

					return video







def 			_SCREAMING_SNAKE_CASE    (       a ,						a=False ,						a=False ,						a=False  )				->				Union[str, Any]:
					assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

					__A     :						List[Any]          =    []
					for i in range(image_processor_tester.batch_size  ):
										if equal_resolution:
															__A     :						int          =    image_processor_tester.max_resolution
										else:
															__A   ,    __A     :						Tuple          =    np.random.choice(
															    np.arange(image_processor_tester.min_resolution ,						image_processor_tester.max_resolution  ) ,						2  )
															__A     :						Optional[int]          =    prepare_video(
															    image_processor_tester=a ,						width=a ,						height=a ,						numpify=a ,						torchify=a ,						)
										video_inputs.append(a  )

					return video_inputs





class 						_A( unittest.TestCase       ):


				"""simple docstring"""


				def __init__(      self					, _A					, _A=7					, _A=3					, _A=4					, _A=18					, _A=30					, _A=400					, _A=True					, _A=None					, _A=True					, _A=[0.5, 0.5, 0.5]					, _A=[0.5, 0.5, 0.5]					, _A=True					, _A=None					, ):
									__A     :						List[str]          =    size if size is not None else {'shortest_edge': 18}
									__A     :						Tuple          =    crop_size if crop_size is not None else {'height': 18, 'width': 18}

									__A     :						Dict          =    parent
									__A     :						str          =    batch_size
									__A     :						int          =    num_channels
									__A     :						Optional[Any]          =    num_frames
									__A     :						Any          =    image_size
									__A     :						List[str]          =    min_resolution
									__A     :						List[Any]          =    max_resolution
									__A     :						Tuple          =    do_resize
									__A     :						List[str]          =    size
									__A     :						str          =    do_normalize
									__A     :						str          =    image_mean
									__A     :						int          =    image_std
									__A     :						Optional[Any]          =    do_center_crop
									__A     :						Union[str, Any]          =    crop_size





				def  UpperCAmelCase_		(      self					):
									return {
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_normalize": self.do_normalize,
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_center_crop": self.do_center_crop,
									    "crop_size": self.crop_size,
									}





@require_torch
@require_vision
class 						_A( snake_case__    ,			unittest.TestCase       ):


				"""simple docstring"""


				UpperCamelCase   :  Optional[Any]    =     TvltImageProcessor if is_vision_available() else None



				def  UpperCAmelCase_		(      self					):
									__A     :						str          =    TvltImageProcessorTester(self					)



				@property
				def  UpperCAmelCase_		(      self					):
									return self.image_processor_tester.prepare_image_processor_dict()



				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.image_processing_class(**self.image_processor_dict					)
									self.assertTrue(hasattr(_A					, 'image_mean'					)					)
									self.assertTrue(hasattr(_A					, 'image_std'					)					)
									self.assertTrue(hasattr(_A					, 'do_normalize'					)					)
									self.assertTrue(hasattr(_A					, 'do_resize'					)					)
									self.assertTrue(hasattr(_A					, 'do_center_crop'					)					)
									self.assertTrue(hasattr(_A					, 'size'					)					)



				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						List[str]          =    self.image_processing_class(**self.image_processor_dict					)
									# create random PIL videos
									__A     :						List[str]          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, Image.Image					)

									# Test not batched input
									__A     :						str          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						List[Any]          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )



				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						Any          =    self.image_processing_class(**self.image_processor_dict					)
									# create random numpy tensors
									__A     :						Dict          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					, numpify=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, np.ndarray					)

									# Test not batched input
									__A     :						Dict          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						Optional[int]          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )





				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						Any          =    self.image_processing_class(**self.image_processor_dict					)
									# create random PyTorch tensors
									__A     :						Dict          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					, torchify=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, torch.Tensor					)

									# Test not batched input
									__A     :						Optional[Any]          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						int          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )




