

import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
	    RwkvForCausalLM,
	    RwkvModel,
	)
	from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
	__snake_case							:     Any 						=	False



class    A__:
						"""simple docstring"""






						def __init__(  self       ,     _lowercase       ,     _lowercase=14       ,     _lowercase=7       ,     _lowercase=True       ,     _lowercase=False       ,     _lowercase=True       ,     _lowercase=True       ,     _lowercase=True       ,     _lowercase=99       ,     _lowercase=32       ,     _lowercase=5       ,     _lowercase=37       ,     _lowercase="gelu"       ,     _lowercase=0.1       ,     _lowercase=0.1       ,     _lowercase=512       ,     _lowercase=16       ,     _lowercase=2       ,     _lowercase=3       ,     _lowercase=4       ,     _lowercase=None       ,     )      ->				List[str]:
									a_		:		Any				    =				parent
									a_		:		Union[str, Any]				    =				batch_size
									a_		:		List[str]				    =				seq_length
									a_		:		List[Any]				    =				is_training
									a_		:		List[Any]				    =				use_token_type_ids
									a_		:		int				    =				use_input_mask
									a_		:		Tuple				    =				use_labels
									a_		:		int				    =				use_mc_token_ids
									a_		:		List[Any]				    =				vocab_size
									a_		:		List[str]				    =				hidden_size
									a_		:		str				    =				num_hidden_layers
									a_		:		Tuple				    =				intermediate_size
									a_		:		Optional[Any]				    =				hidden_act
									a_		:		Tuple				    =				hidden_dropout_prob
									a_		:		Tuple				    =				attention_probs_dropout_prob
									a_		:		Any				    =				max_position_embeddings
									a_		:		Union[str, Any]				    =				type_vocab_size
									a_		:		List[Any]				    =				type_sequence_label_size
									a_		:		Tuple				    =				num_labels
									a_		:		Optional[Any]				    =				num_choices
									a_		:		Optional[int]				    =				scope
									a_		:		str				    =				vocab_size - 1
									a_		:		int				    =				vocab_size - 1
									a_		:		List[str]				    =				vocab_size - 1





						def 	UpperCamelCase__			(  self  )      ->				Optional[Any]:
									return RwkvConfig.from_pretrained("""sgugger/rwkv-4-pile-7b"""  )





						def 	UpperCamelCase__			(  self       ,     _lowercase=False       ,     _lowercase=False       ,     _lowercase=False  )      ->				List[Any]:
									a_		:		str				    =				ids_tensor([self.batch_size, self.seq_length]       ,     self.vocab_size  )

									a_		:		Union[str, Any]				    =				None
									if self.use_input_mask:
												a_		:		List[str]				    =				random_attention_mask([self.batch_size, self.seq_length]  )

									a_		:		int				    =				None
									if self.use_token_type_ids:
												a_		:		Any				    =				ids_tensor([self.batch_size, self.seq_length]       ,     self.type_vocab_size  )

									a_		:		int				    =				None
									if self.use_mc_token_ids:
												a_		:		List[Any]				    =				ids_tensor([self.batch_size, self.num_choices]       ,     self.seq_length  )

									a_		:		Any				    =				None
									a_		:		Union[str, Any]				    =				None
									a_		:		Dict				    =				None
									if self.use_labels:
												a_		:		Tuple				    =				ids_tensor([self.batch_size]       ,     self.type_sequence_label_size  )
												a_		:		str				    =				ids_tensor([self.batch_size, self.seq_length]       ,     self.num_labels  )
												a_		:		List[Any]				    =				ids_tensor([self.batch_size]       ,     self.num_choices  )

									a_		:		int				    =				self.get_config(
									    gradient_checkpointing=_lowercase       ,     scale_attn_by_inverse_layer_idx=_lowercase       ,     reorder_and_upcast_attn=_lowercase       ,     )

									return (
									    config,
									    input_ids,
									    input_mask,
									    None,
									    token_type_ids,
									    mc_token_ids,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									)





						def 	UpperCamelCase__			(  self       ,     _lowercase=False       ,     _lowercase=False       ,     _lowercase=False  )      ->				Any:
									return RwkvConfig(
									    vocab_size=self.vocab_size       ,     hidden_size=self.hidden_size       ,     num_hidden_layers=self.num_hidden_layers       ,     intermediate_size=self.intermediate_size       ,     activation_function=self.hidden_act       ,     resid_pdrop=self.hidden_dropout_prob       ,     attn_pdrop=self.attention_probs_dropout_prob       ,     n_positions=self.max_position_embeddings       ,     type_vocab_size=self.type_vocab_size       ,     use_cache=_lowercase       ,     bos_token_id=self.bos_token_id       ,     eos_token_id=self.eos_token_id       ,     pad_token_id=self.pad_token_id       ,     gradient_checkpointing=_lowercase       ,     scale_attn_by_inverse_layer_idx=_lowercase       ,     reorder_and_upcast_attn=_lowercase       ,     )





						def 	UpperCamelCase__			(  self  )      ->				Tuple:
									a_		:		str				    =				self.get_config()
									a_		:		Any				    =				300
									return config





						def 	UpperCamelCase__			(  self  )      ->				str:
									(
									    (
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			
									)		:		Dict				    =				self.prepare_config_and_inputs()

									a_		:		Tuple				    =				floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
									a_		:		List[str]				    =				ids_tensor([self.batch_size, self.seq_length]       ,     vocab_size=2  )

									return (
									    config,
									    input_ids,
									    input_mask,
									    head_mask,
									    token_type_ids,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    encoder_hidden_states,
									    encoder_attention_mask,
									)





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     *_lowercase  )      ->				List[Any]:
									a_		:		Optional[Any]				    =				True
									a_		:		Union[str, Any]				    =				RwkvModel(config=_lowercase  )
									model.to(_lowercase  )
									model.eval()

									a_		:		int				    =				model(_lowercase  )

									self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)  )
									self.parent.assertEqual(len(result.hidden_states  )       ,     config.num_hidden_layers + 1  )





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     *_lowercase  )      ->				int:
									a_		:		List[str]				    =				RwkvForCausalLM(_lowercase  )
									model.to(_lowercase  )
									model.eval()

									a_		:		Tuple				    =				model(_lowercase       ,     labels=_lowercase  )
									self.parent.assertEqual(result.loss.shape       ,     ()  )
									self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.vocab_size)  )





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     *_lowercase  )      ->				List[Any]:
									a_		:		Any				    =				RwkvModel(config=_lowercase  )
									model.to(_lowercase  )
									model.eval()

									a_		:		Any				    =				model(_lowercase  )
									a_		:		Tuple				    =				outputs.last_hidden_state

									a_		:		str				    =				model(input_ids[:, :2]  )
									a_		:		List[str]				    =				outputs.last_hidden_state

									# Using the state computed on the first inputs, we will get the same output
									a_		:		Tuple				    =				model(input_ids[:, 2:]       ,     state=outputs.state  )
									a_		:		Tuple				    =				outputs.last_hidden_state

									self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]       ,     dim=1  )       ,     _lowercase       ,     atol=1e-5  )  )





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     _lowercase       ,     *_lowercase       ,     _lowercase=False  )      ->				List[Any]:
									a_		:		Union[str, Any]				    =				RwkvForCausalLM(_lowercase  )
									model.to(_lowercase  )
									if gradient_checkpointing:
												model.gradient_checkpointing_enable()

									a_		:		int				    =				model(_lowercase       ,     labels=_lowercase  )
									self.parent.assertEqual(result.loss.shape       ,     ()  )
									self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.vocab_size)  )
									result.loss.backward()





						def 	UpperCamelCase__			(  self  )      ->				List[str]:
									a_		:		Tuple				    =				self.prepare_config_and_inputs()

									(
									    (
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			(
									    a_
									)    ,			
									)		:		List[str]				    =				config_and_inputs

									a_		:		Tuple				    =				{"""input_ids""": input_ids}

									return config, inputs_dict







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,       reason='''See https://github.com/huggingface/transformers/pull/24204'''    )
@require_torch
class    A__(a_,       a_,       a_,       unittest.TestCase    ):
						"""simple docstring"""






						_A  :							List[Any]   			=	(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
						_A  :							List[Any]   			=	(
						    {'''feature-extraction''': RwkvModel, '''text-generation''': RwkvForCausalLM} if is_torch_available() else {}
						)
						# all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
						_A  :							Any   			=	False
						_A  :							Tuple   			=	False
						_A  :							Union[str, Any]   			=	False
						_A  :							Optional[int]   			=	False
						_A  :							Any   			=	False  # Rwkv does not support head masking





						def 	UpperCamelCase__			(  self  )      ->				int:
									a_		:		Any				    =				RwkvModelTester(self  )
									a_		:		Union[str, Any]				    =				ConfigTester(
									    self       ,     config_class=_lowercase       ,     n_embd=37       ,     common_properties=["""hidden_size""", """num_hidden_layers"""]  )





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase=None  )      ->				Any:
									if isinstance(_lowercase       ,     torch.Tensor  ):
												a_    ,			a_		:		List[Any]				    =				member.max().item(), member.min().item()
									elif isinstance(_lowercase       ,     _lowercase  ) or isinstance(_lowercase       ,     _lowercase  ):
												a_    ,			a_		:		Optional[Any]				    =				max(_lowercase  ), min(_lowercase  )

									if not isinstance(_lowercase       ,     _lowercase  ):
												raise TypeError("""container should be a list or tuple"""  )
									elif len(_lowercase  ) != 2:
												raise ValueError("""container should have 2 elements"""  )

									a_    ,			a_		:		Union[str, Any]				    =				container

									a_		:		Any				    =				(min_value >= expected_min) and (max_value <= expected_max)

									if not is_inside_interval:
												a_		:		Optional[int]				    =				"""%s not found in %s""" % (safe_repr(_lowercase  ), safe_repr(_lowercase  ))
												self.fail(self._formatMessage(_lowercase       ,     _lowercase  )  )





						def 	UpperCamelCase__			(  self  )      ->				Any:
									self.config_tester.run_common_tests()





						def 	UpperCamelCase__			(  self  )      ->				Optional[Any]:
									a_		:		List[str]				    =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_rwkv_model(*_lowercase  )





						def 	UpperCamelCase__			(  self  )      ->				Tuple:
									a_		:		Any				    =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_causl_lm(*_lowercase  )





						def 	UpperCamelCase__			(  self  )      ->				Tuple:
									a_		:		Optional[Any]				    =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_state_equivalency(*_lowercase  )





						def 	UpperCamelCase__			(  self  )      ->				Optional[int]:
									a_    ,			a_		:		Optional[Any]				    =				self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
												a_		:		Any				    =				model_class(config=_lowercase  )
												for name, param in model.named_parameters():
															if "time_decay" in name:
																		if param.requires_grad:
																					self.assertTrue(param.data.max().item() == 3.0  )
																					self.assertTrue(param.data.min().item() == -5.0  )
															elif "time_first" in name:
																		if param.requires_grad:
																					# check if it's a ones like
																					self.assertTrue(torch.allclose(param.data       ,     torch.ones_like(param.data  )       ,     atol=1e-5       ,     rtol=1e-5  )  )
															elif any(x in name for x in ["""time_mix_key""", """time_mix_receptance"""]  ):
																		if param.requires_grad:
																					self.assertInterval(
																					    param.data       ,     [0.0, 1.0]       ,     msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''       ,     )
															elif "time_mix_value" in name:
																		if param.requires_grad:
																					self.assertInterval(
																					    param.data       ,     [0.0, 1.3]       ,     msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''       ,     )





						def 	UpperCamelCase__			(  self  )      ->				Optional[int]:
									a_    ,			a_		:		Dict				    =				self.model_tester.prepare_config_and_inputs_for_common()
									a_		:		Dict				    =				True

									a_		:		str				    =				getattr(self.model_tester       ,     """seq_length"""       ,     _lowercase  )

									for model_class in self.all_model_classes:
												a_		:		List[Any]				    =				True
												a_		:		Optional[int]				    =				False
												a_		:		List[str]				    =				True
												a_		:		Optional[int]				    =				model_class(_lowercase  )
												model.to(_lowercase  )
												model.eval()

												a_		:		Optional[int]				    =				self._prepare_for_class(_lowercase       ,     _lowercase  )
												a_		:		List[Any]				    =				inputs["""input_ids"""].shape[0]
												with torch.no_grad():
															a_		:		List[str]				    =				model(**_lowercase  )
												a_		:		Optional[Any]				    =				outputs.attentions
												self.assertEqual(len(_lowercase  )       ,     self.model_tester.num_hidden_layers  )

												# check that output_attentions also work using config
												del inputs_dict["output_attentions"]
												a_		:		Any				    =				True
												a_		:		Union[str, Any]				    =				model_class(_lowercase  )
												model.to(_lowercase  )
												model.eval()

												a_		:		Any				    =				self._prepare_for_class(_lowercase       ,     _lowercase  )
												a_		:		int				    =				inputs["""input_ids"""].shape[0]
												with torch.no_grad():
															a_		:		Optional[Any]				    =				model(**_lowercase  )
												a_		:		Dict				    =				outputs.attentions
												self.assertEqual(len(_lowercase  )       ,     self.model_tester.num_hidden_layers  )

												self.assertListEqual(
												    list(attentions[0].shape[-3:]  )       ,     [batch_size, seq_len, config.hidden_size]       ,     )
												a_		:		Dict				    =				len(_lowercase  )

												# Check attention is always last and order is fine
												a_		:		Dict				    =				True
												a_		:		Any				    =				True
												a_		:		Union[str, Any]				    =				model_class(_lowercase  )
												model.to(_lowercase  )
												model.eval()

												a_		:		Union[str, Any]				    =				self._prepare_for_class(_lowercase       ,     _lowercase  )
												a_		:		Dict				    =				inputs["""input_ids"""].shape[0]
												with torch.no_grad():
															a_		:		List[str]				    =				model(**_lowercase  )

												a_		:		int				    =				1
												self.assertEqual(out_len + added_hidden_states       ,     len(_lowercase  )  )

												a_		:		Tuple				    =				outputs.attentions

												self.assertEqual(len(_lowercase  )       ,     self.model_tester.num_hidden_layers  )
												self.assertListEqual(
												    list(self_attentions[0].shape[-3:]  )       ,     [batch_size, seq_len, config.hidden_size]       ,     )





						@slow
						def 	UpperCamelCase__			(  self  )      ->				str:
									for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												a_		:		Dict				    =				RwkvModel.from_pretrained(_lowercase  )
												self.assertIsNotNone(_lowercase  )







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,       reason='''See https://github.com/huggingface/transformers/pull/24204'''    )
@slow
class    A__(unittest.TestCase    ):
						"""simple docstring"""






						def 	UpperCamelCase__			(  self  )      ->				Tuple:
									a_		:		str				    =				"""RWKV/rwkv-4-169m-pile"""
									a_		:		int				    =				AutoTokenizer.from_pretrained(self.model_id  )





						def 	UpperCamelCase__			(  self  )      ->				Any:
									a_		:		Union[str, Any]				    =				"""Hello my name is Jasmine and I am a newbie to the"""
									a_		:		Optional[Any]				    =				RwkvForCausalLM.from_pretrained(self.model_id  ).to(_lowercase  )

									a_		:		List[str]				    =				self.tokenizer("""Hello my name is"""       ,     return_tensors="""pt"""  ).input_ids.to(_lowercase  )
									a_		:		Any				    =				model.generate(_lowercase       ,     max_new_tokens=10  )
									a_		:		List[str]				    =				self.tokenizer.decode(output[0].tolist()  )

									self.assertEqual(_lowercase       ,     _lowercase  )





						def 	UpperCamelCase__			(  self  )      ->				str:
									a_		:		int				    =				"""Hello my name is Jasmine and I am a newbie to the"""

									a_		:		List[str]				    =				self.tokenizer("""Hello my name is"""       ,     return_tensors="""pt"""  ).input_ids.to(_lowercase  )
									a_		:		List[Any]				    =				RwkvForCausalLM.from_pretrained(self.model_id       ,     torch_dtype=torch.bfloataa  ).to(_lowercase  )

									a_		:		Dict				    =				model.generate(_lowercase       ,     max_new_tokens=10  )
									a_		:		Optional[Any]				    =				self.tokenizer.decode(output[0].tolist()  )

									self.assertEqual(_lowercase       ,     _lowercase  )





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							:     Optional[Any] 						=	logging.get_logger(__name__)

__snake_case							:     str 						=	{
    """ZinengTang/tvlt-base""": """https://huggingface.co/ZinengTang/tvlt-base/blob/main/config.json""",
}



class    A__(a_    ):
						"""simple docstring"""






						_A  :							Any   			=	'''tvlt'''





						def __init__(  self       ,     _lowercase=224       ,     _lowercase=2_048       ,     _lowercase=128       ,     _lowercase=[16, 16]       ,     _lowercase=[16, 16]       ,     _lowercase=3       ,     _lowercase=1       ,     _lowercase=8       ,     _lowercase=768       ,     _lowercase=12       ,     _lowercase=12       ,     _lowercase=3_072       ,     _lowercase="gelu"       ,     _lowercase=0.0       ,     _lowercase=0.0       ,     _lowercase=0.0_2       ,     _lowercase=1e-6       ,     _lowercase=True       ,     _lowercase=False       ,     _lowercase=16       ,     _lowercase=512       ,     _lowercase=8       ,     _lowercase=2_048       ,     _lowercase=0.7_5       ,     _lowercase=0.1_5       ,     _lowercase="frame-level"       ,     _lowercase=True       ,     _lowercase=True       ,     _lowercase="classification"       ,     **_lowercase       ,     )      ->				Tuple:
									super().__init__(**_lowercase  )

									if audio_mask_type not in ("frame-level", "patch_level"):
												raise ValueError(
												    """audio_mask_type must be one of two acceptable strategies - {'frame_level', 'patch-level') """
												    F'''got {audio_mask_type}'''  )

									a_		:		int				    =				image_size
									a_		:		str				    =				spectrogram_length
									a_		:		int				    =				frequency_length
									a_		:		str				    =				image_patch_size
									a_		:		int				    =				audio_patch_size
									a_		:		Any				    =				num_image_channels
									a_		:		str				    =				num_audio_channels
									a_		:		int				    =				num_frames

									a_		:		Optional[int]				    =				hidden_size
									a_		:		Tuple				    =				num_hidden_layers
									a_		:		Any				    =				num_attention_heads
									a_		:		Union[str, Any]				    =				intermediate_size
									a_		:		Tuple				    =				hidden_act
									a_		:		Union[str, Any]				    =				hidden_dropout_prob
									a_		:		int				    =				attention_probs_dropout_prob
									a_		:		Any				    =				initializer_range
									a_		:		int				    =				layer_norm_eps
									a_		:		Union[str, Any]				    =				qkv_bias
									a_		:		str				    =				use_mean_pooling

									a_		:		List[Any]				    =				decoder_num_attention_heads
									a_		:		Optional[Any]				    =				decoder_hidden_size
									a_		:		Dict				    =				decoder_num_hidden_layers
									a_		:		Optional[Any]				    =				decoder_intermediate_size
									a_		:		Dict				    =				pixel_mask_ratio
									a_		:		int				    =				audio_mask_ratio
									a_		:		Optional[Any]				    =				audio_mask_type

									a_		:		Union[str, Any]				    =				task_matching
									a_		:		Dict				    =				task_mae
									a_		:		Union[str, Any]				    =				loss_type


