


import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
							    RwkvForCausalLM,
							    RwkvModel,
							)
							from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
							__magic_name__       					=      False





class    lowerCAmelCase__				:






						"""simple docstring"""




						def __init__(   self				,							a_				,							a_=14				,							a_=7				,							a_=True				,							a_=False				,							a_=True				,							a_=True				,							a_=True				,							a_=99				,							a_=32				,							a_=5				,							a_=37				,							a_="gelu"				,							a_=0.1				,							a_=0.1				,							a_=512				,							a_=16				,							a_=2				,							a_=3				,							a_=4				,							a_=None				,							):
							lowerCamelCase_			:							List[str]      =       parent
							lowerCamelCase_			:							int      =       batch_size
							lowerCamelCase_			:							int      =       seq_length
							lowerCamelCase_			:							Union[str, Any]      =       is_training
							lowerCamelCase_			:							Any      =       use_token_type_ids
							lowerCamelCase_			:							List[str]      =       use_input_mask
							lowerCamelCase_			:							Any      =       use_labels
							lowerCamelCase_			:							Optional[int]      =       use_mc_token_ids
							lowerCamelCase_			:							Optional[Any]      =       vocab_size
							lowerCamelCase_			:							Tuple      =       hidden_size
							lowerCamelCase_			:							List[Any]      =       num_hidden_layers
							lowerCamelCase_			:							List[str]      =       intermediate_size
							lowerCamelCase_			:							List[Any]      =       hidden_act
							lowerCamelCase_			:							Optional[int]      =       hidden_dropout_prob
							lowerCamelCase_			:							List[Any]      =       attention_probs_dropout_prob
							lowerCamelCase_			:							List[str]      =       max_position_embeddings
							lowerCamelCase_			:							Dict      =       type_vocab_size
							lowerCamelCase_			:							Optional[int]      =       type_sequence_label_size
							lowerCamelCase_			:							List[str]      =       num_labels
							lowerCamelCase_			:							Optional[Any]      =       num_choices
							lowerCamelCase_			:							Any      =       scope
							lowerCamelCase_			:							str      =       vocab_size - 1
							lowerCamelCase_			:							Optional[int]      =       vocab_size - 1
							lowerCamelCase_			:							Optional[Any]      =       vocab_size - 1




						def 					_UpperCamelCase			(   self     ):
							return RwkvConfig.from_pretrained("sgugger/rwkv-4-pile-7b"     )




						def 					_UpperCamelCase			(   self				,							a_=False				,							a_=False				,							a_=False     ):
							lowerCamelCase_			:							List[Any]      =       ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size     )

							lowerCamelCase_			:							Dict      =       None
							if self.use_input_mask:
								lowerCamelCase_			:							Tuple      =       random_attention_mask([self.batch_size, self.seq_length]     )

							lowerCamelCase_			:							str      =       None
							if self.use_token_type_ids:
								lowerCamelCase_			:							List[str]      =       ids_tensor([self.batch_size, self.seq_length]				,							self.type_vocab_size     )

							lowerCamelCase_			:							Optional[int]      =       None
							if self.use_mc_token_ids:
								lowerCamelCase_			:							Any      =       ids_tensor([self.batch_size, self.num_choices]				,							self.seq_length     )

							lowerCamelCase_			:							Union[str, Any]      =       None
							lowerCamelCase_			:							str      =       None
							lowerCamelCase_			:							Any      =       None
							if self.use_labels:
								lowerCamelCase_			:							int      =       ids_tensor([self.batch_size]				,							self.type_sequence_label_size     )
								lowerCamelCase_			:							Dict      =       ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels     )
								lowerCamelCase_			:							Dict      =       ids_tensor([self.batch_size]				,							self.num_choices     )

							lowerCamelCase_			:							Optional[int]      =       self.get_config(
							    gradient_checkpointing=a_				,							scale_attn_by_inverse_layer_idx=a_				,							reorder_and_upcast_attn=a_				,							)

							return (
							    config,
							    input_ids,
							    input_mask,
							    None,
							    token_type_ids,
							    mc_token_ids,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							)




						def 					_UpperCamelCase			(   self				,							a_=False				,							a_=False				,							a_=False     ):
							return RwkvConfig(
							    vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							intermediate_size=self.intermediate_size				,							activation_function=self.hidden_act				,							resid_pdrop=self.hidden_dropout_prob				,							attn_pdrop=self.attention_probs_dropout_prob				,							n_positions=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							use_cache=a_				,							bos_token_id=self.bos_token_id				,							eos_token_id=self.eos_token_id				,							pad_token_id=self.pad_token_id				,							gradient_checkpointing=a_				,							scale_attn_by_inverse_layer_idx=a_				,							reorder_and_upcast_attn=a_				,							)




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Optional[Any]      =       self.get_config()
							lowerCamelCase_			:							Dict      =       300
							return config




						def 					_UpperCamelCase			(   self     ):
							(
							    (
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,
							)			:							Optional[int]      =       self.prepare_config_and_inputs()

							lowerCamelCase_			:							Any      =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
							lowerCamelCase_			:							Union[str, Any]      =       ids_tensor([self.batch_size, self.seq_length]				,							vocab_size=2     )

							return (
							    config,
							    input_ids,
							    input_mask,
							    head_mask,
							    token_type_ids,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    encoder_hidden_states,
							    encoder_attention_mask,
							)




						def 					_UpperCamelCase			(   self				,							a_				,							a_				,							a_				,							a_				,							a_				,							*a_     ):
							lowerCamelCase_			:							List[str]      =       True
							lowerCamelCase_			:							List[str]      =       RwkvModel(config=a_     )
							model.to(a_     )
							model.eval()

							lowerCamelCase_			:							Union[str, Any]      =       model(a_     )

							self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)     )
							self.parent.assertEqual(len(result.hidden_states     )				,							config.num_hidden_layers + 1     )




						def 					_UpperCamelCase			(   self				,							a_				,							a_				,							a_				,							a_				,							a_				,							*a_     ):
							lowerCamelCase_			:							List[Any]      =       RwkvForCausalLM(a_     )
							model.to(a_     )
							model.eval()

							lowerCamelCase_			:							Any      =       model(a_				,							labels=a_     )
							self.parent.assertEqual(result.loss.shape				,							()     )
							self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)     )




						def 					_UpperCamelCase			(   self				,							a_				,							a_				,							a_				,							a_				,							a_				,							*a_     ):
							lowerCamelCase_			:							List[str]      =       RwkvModel(config=a_     )
							model.to(a_     )
							model.eval()

							lowerCamelCase_			:							List[str]      =       model(a_     )
							lowerCamelCase_			:							Dict      =       outputs.last_hidden_state

							lowerCamelCase_			:							Union[str, Any]      =       model(input_ids[:, :2]     )
							lowerCamelCase_			:							str      =       outputs.last_hidden_state

							# Using the state computed on the first inputs, we will get the same output
							lowerCamelCase_			:							Optional[Any]      =       model(input_ids[:, 2:]				,							state=outputs.state     )
							lowerCamelCase_			:							List[Any]      =       outputs.last_hidden_state

							self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]				,							dim=1     )				,							a_				,							atol=1E-5     )     )




						def 					_UpperCamelCase			(   self				,							a_				,							a_				,							a_				,							a_				,							a_				,							*a_				,							a_=False     ):
							lowerCamelCase_			:							Optional[int]      =       RwkvForCausalLM(a_     )
							model.to(a_     )
							if gradient_checkpointing:
								model.gradient_checkpointing_enable()

							lowerCamelCase_			:							List[Any]      =       model(a_				,							labels=a_     )
							self.parent.assertEqual(result.loss.shape				,							()     )
							self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)     )
							result.loss.backward()




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Dict      =       self.prepare_config_and_inputs()

							(
							    (
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,(
							    lowerCamelCase_
							)   ,
							)			:							Any      =       config_and_inputs

							lowerCamelCase_			:							Any      =       {"input_ids": input_ids}

							return config, inputs_dict





@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,    reason='''See https://github.com/huggingface/transformers/pull/24204'''       )
@require_torch
class    lowerCAmelCase__				(			__lowerCamelCase,    __lowerCamelCase,    __lowerCamelCase,    unittest.TestCase       ):






						"""simple docstring"""
						__UpperCAmelCase   :		List[Any]  					=	(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
						__UpperCAmelCase   :		List[Any]  					=	(
						    {'''feature-extraction''': RwkvModel, '''text-generation''': RwkvForCausalLM} if is_torch_available() else {}
						)
						# all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
						__UpperCAmelCase   :		Union[str, Any]  					=	False
						__UpperCAmelCase   :		Tuple  					=	False
						__UpperCAmelCase   :		str  					=	False
						__UpperCAmelCase   :		Optional[int]  					=	False
						__UpperCAmelCase   :		int  					=	False  # Rwkv does not support head masking




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Dict      =       RwkvModelTester(self     )
							lowerCamelCase_			:							List[Any]      =       ConfigTester(
							    self				,							config_class=a_				,							n_embd=37				,							common_properties=["hidden_size", "num_hidden_layers"]     )




						def 					_UpperCamelCase			(   self				,							a_				,							a_				,							a_=None     ):
							if isinstance(a_				,							torch.Tensor     ):
								lowerCamelCase_   ,lowerCamelCase_			:							List[Any]      =       member.max().item(), member.min().item()
							elif isinstance(a_				,							a_     ) or isinstance(a_				,							a_     ):
								lowerCamelCase_   ,lowerCamelCase_			:							Union[str, Any]      =       max(a_     ), min(a_     )

							if not isinstance(a_				,							a_     ):
								raise TypeError("container should be a list or tuple"     )
							elif len(a_     ) != 2:
								raise ValueError("container should have 2 elements"     )

							lowerCamelCase_   ,lowerCamelCase_			:							Union[str, Any]      =       container

							lowerCamelCase_			:							Optional[Any]      =       (min_value >= expected_min) and (max_value <= expected_max)

							if not is_inside_interval:
								lowerCamelCase_			:							Any      =       "%s not found in %s" % (safe_repr(a_     ), safe_repr(a_     ))
								self.fail(self._formatMessage(a_				,							a_     )     )




						def 					_UpperCamelCase			(   self     ):
							self.config_tester.run_common_tests()




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Optional[int]      =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_rwkv_model(*a_     )




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Tuple      =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_causl_lm(*a_     )




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Tuple      =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_state_equivalency(*a_     )




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_   ,lowerCamelCase_			:							Union[str, Any]      =       self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								lowerCamelCase_			:							Optional[int]      =       model_class(config=a_     )
								for name, param in model.named_parameters():
									if "time_decay" in name:
										if param.requires_grad:
											self.assertTrue(param.data.max().item() == 3.0     )
											self.assertTrue(param.data.min().item() == -5.0     )
									elif "time_first" in name:
										if param.requires_grad:
											# check if it's a ones like
											self.assertTrue(torch.allclose(param.data				,							torch.ones_like(param.data     )				,							atol=1E-5				,							rtol=1E-5     )     )
									elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]     ):
										if param.requires_grad:
											self.assertInterval(
											    param.data				,							[0.0, 1.0]				,							msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""				,							)
									elif "time_mix_value" in name:
										if param.requires_grad:
											self.assertInterval(
											    param.data				,							[0.0, 1.3]				,							msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""				,							)




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_   ,lowerCamelCase_			:							List[Any]      =       self.model_tester.prepare_config_and_inputs_for_common()
							lowerCamelCase_			:							List[str]      =       True

							lowerCamelCase_			:							Tuple      =       getattr(self.model_tester				,							"seq_length"				,							a_     )

							for model_class in self.all_model_classes:
								lowerCamelCase_			:							int      =       True
								lowerCamelCase_			:							str      =       False
								lowerCamelCase_			:							str      =       True
								lowerCamelCase_			:							str      =       model_class(a_     )
								model.to(a_     )
								model.eval()

								lowerCamelCase_			:							Dict      =       self._prepare_for_class(a_				,							a_     )
								lowerCamelCase_			:							Any      =       inputs["input_ids"].shape[0]
								with torch.no_grad():
									lowerCamelCase_			:							str      =       model(**a_     )
								lowerCamelCase_			:							Optional[Any]      =       outputs.attentions
								self.assertEqual(len(a_     )				,							self.model_tester.num_hidden_layers     )

								# check that output_attentions also work using config
								del inputs_dict["output_attentions"]
								lowerCamelCase_			:							int      =       True
								lowerCamelCase_			:							Dict      =       model_class(a_     )
								model.to(a_     )
								model.eval()

								lowerCamelCase_			:							Dict      =       self._prepare_for_class(a_				,							a_     )
								lowerCamelCase_			:							Dict      =       inputs["input_ids"].shape[0]
								with torch.no_grad():
									lowerCamelCase_			:							Optional[Any]      =       model(**a_     )
								lowerCamelCase_			:							int      =       outputs.attentions
								self.assertEqual(len(a_     )				,							self.model_tester.num_hidden_layers     )

								self.assertListEqual(
								    list(attentions[0].shape[-3:]     )				,							[batch_size, seq_len, config.hidden_size]				,							)
								lowerCamelCase_			:							Tuple      =       len(a_     )

								# Check attention is always last and order is fine
								lowerCamelCase_			:							List[Any]      =       True
								lowerCamelCase_			:							List[Any]      =       True
								lowerCamelCase_			:							int      =       model_class(a_     )
								model.to(a_     )
								model.eval()

								lowerCamelCase_			:							List[Any]      =       self._prepare_for_class(a_				,							a_     )
								lowerCamelCase_			:							List[Any]      =       inputs["input_ids"].shape[0]
								with torch.no_grad():
									lowerCamelCase_			:							Union[str, Any]      =       model(**a_     )

								lowerCamelCase_			:							Optional[int]      =       1
								self.assertEqual(out_len + added_hidden_states				,							len(a_     )     )

								lowerCamelCase_			:							List[str]      =       outputs.attentions

								self.assertEqual(len(a_     )				,							self.model_tester.num_hidden_layers     )
								self.assertListEqual(
								    list(self_attentions[0].shape[-3:]     )				,							[batch_size, seq_len, config.hidden_size]				,							)




						@slow
						def 					_UpperCamelCase			(   self     ):
							for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								lowerCamelCase_			:							int      =       RwkvModel.from_pretrained(a_     )
								self.assertIsNotNone(a_     )





@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,    reason='''See https://github.com/huggingface/transformers/pull/24204'''       )
@slow
class    lowerCAmelCase__				(			unittest.TestCase       ):






						"""simple docstring"""




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							str      =       "RWKV/rwkv-4-169m-pile"
							lowerCamelCase_			:							Dict      =       AutoTokenizer.from_pretrained(self.model_id     )




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							int      =       "Hello my name is Jasmine and I am a newbie to the"
							lowerCamelCase_			:							Union[str, Any]      =       RwkvForCausalLM.from_pretrained(self.model_id     ).to(a_     )

							lowerCamelCase_			:							List[Any]      =       self.tokenizer("Hello my name is"				,							return_tensors="pt"     ).input_ids.to(a_     )
							lowerCamelCase_			:							str      =       model.generate(a_				,							max_new_tokens=10     )
							lowerCamelCase_			:							Tuple      =       self.tokenizer.decode(output[0].tolist()     )

							self.assertEqual(a_				,							a_     )




						def 					_UpperCamelCase			(   self     ):
							lowerCamelCase_			:							Dict      =       "Hello my name is Jasmine and I am a newbie to the"

							lowerCamelCase_			:							int      =       self.tokenizer("Hello my name is"				,							return_tensors="pt"     ).input_ids.to(a_     )
							lowerCamelCase_			:							Tuple      =       RwkvForCausalLM.from_pretrained(self.model_id				,							torch_dtype=torch.bfloataa     ).to(a_     )

							lowerCamelCase_			:							List[Any]      =       model.generate(a_				,							max_new_tokens=10     )
							lowerCamelCase_			:							Dict      =       self.tokenizer.decode(output[0].tolist()     )

							self.assertEqual(a_				,							a_     )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__magic_name__       					=      {
    '''configuration_big_bird''': ['''BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BigBirdConfig''', '''BigBirdOnnxConfig'''],
}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__magic_name__       					=      ['''BigBirdTokenizer''']

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__magic_name__       					=      ['''BigBirdTokenizerFast''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__magic_name__       					=      [
							    '''BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''BigBirdForCausalLM''',
							    '''BigBirdForMaskedLM''',
							    '''BigBirdForMultipleChoice''',
							    '''BigBirdForPreTraining''',
							    '''BigBirdForQuestionAnswering''',
							    '''BigBirdForSequenceClassification''',
							    '''BigBirdForTokenClassification''',
							    '''BigBirdLayer''',
							    '''BigBirdModel''',
							    '''BigBirdPreTrainedModel''',
							    '''load_tf_weights_in_big_bird''',
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__magic_name__       					=      [
							    '''FlaxBigBirdForCausalLM''',
							    '''FlaxBigBirdForMaskedLM''',
							    '''FlaxBigBirdForMultipleChoice''',
							    '''FlaxBigBirdForPreTraining''',
							    '''FlaxBigBirdForQuestionAnswering''',
							    '''FlaxBigBirdForSequenceClassification''',
							    '''FlaxBigBirdForTokenClassification''',
							    '''FlaxBigBirdModel''',
							    '''FlaxBigBirdPreTrainedModel''',
							]

if TYPE_CHECKING:
							from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_big_bird import BigBirdTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_big_bird_fast import BigBirdTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_big_bird import (
														    BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
														    BigBirdForCausalLM,
														    BigBirdForMaskedLM,
														    BigBirdForMultipleChoice,
														    BigBirdForPreTraining,
														    BigBirdForQuestionAnswering,
														    BigBirdForSequenceClassification,
														    BigBirdForTokenClassification,
														    BigBirdLayer,
														    BigBirdModel,
														    BigBirdPreTrainedModel,
														    load_tf_weights_in_big_bird,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_big_bird import (
														    FlaxBigBirdForCausalLM,
														    FlaxBigBirdForMaskedLM,
														    FlaxBigBirdForMultipleChoice,
														    FlaxBigBirdForPreTraining,
														    FlaxBigBirdForQuestionAnswering,
														    FlaxBigBirdForSequenceClassification,
														    FlaxBigBirdForTokenClassification,
														    FlaxBigBirdModel,
														    FlaxBigBirdPreTrainedModel,
														)

else:
							import sys

							__magic_name__       					=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

