



'''simple docstring'''





import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
      import jax
      import jax.numpy as jnp

      from transformers.modeling_flax_pytorch_utils import (
          convert_pytorch_state_dict_to_flax,
          load_flax_weights_in_pytorch_model,
      )
      from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
      import torch





class 							lowerCAmelCase_		:





    def __init__(    self    :    Optional[Any],    _snake_case    :    Optional[int],    _snake_case    :    Tuple=14,    _snake_case    :    Tuple=7,    _snake_case    :    int=True,    _snake_case    :    Optional[int]=True,    _snake_case    :    Any=False,    _snake_case    :    Tuple=True,    _snake_case    :    List[Any]=99,    _snake_case    :    Optional[Any]=32,    _snake_case    :    Optional[Any]=5,    _snake_case    :    Any=4,    _snake_case    :    int=37,    _snake_case    :    List[str]="gelu",    _snake_case    :    int=0.1,    _snake_case    :    Dict=0.1,    _snake_case    :    Optional[Any]=512,    _snake_case    :    Any=0.02,    ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =parent
       snake_case   :							Optional[Any]						  =batch_size
       snake_case   :							Union[str, Any]						  =seq_length
       snake_case   :							Any						  =is_training
       snake_case   :							List[Any]						  =use_input_mask
       snake_case   :							int						  =use_token_type_ids
       snake_case   :							Dict						  =use_labels
       snake_case   :							int						  =vocab_size
       snake_case   :							Union[str, Any]						  =hidden_size
       snake_case   :							int						  =num_hidden_layers
       snake_case   :							Optional[Any]						  =num_attention_heads
       snake_case   :							int						  =intermediate_size
       snake_case   :							Optional[Any]						  =hidden_act
       snake_case   :							Dict						  =hidden_dropout_prob
       snake_case   :							str						  =attention_probs_dropout_prob
       snake_case   :							List[str]						  =max_position_embeddings
       snake_case   :							Tuple						  =initializer_range
       snake_case   :							Dict						  =None
       snake_case   :							Any						  =vocab_size - 1
       snake_case   :							Dict						  =vocab_size - 1
       snake_case   :							Tuple						  =vocab_size - 1





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							Tuple						  =ids_tensor([self.batch_size, self.seq_length],    self.vocab_size     )

       snake_case   :							Any						  =None
       if self.use_input_mask:
          snake_case   :							List[Any]						  =random_attention_mask([self.batch_size, self.seq_length]     )

       snake_case   :							str						  =GPTaConfig(
           vocab_size=self.vocab_size,    n_embd=self.hidden_size,    n_layer=self.num_hidden_layers,    n_head=self.num_attention_heads,    n_positions=self.max_position_embeddings,    use_cache=_snake_case,    bos_token_id=self.bos_token_id,    eos_token_id=self.eos_token_id,    pad_token_id=self.pad_token_id,    )

       return (config, input_ids, input_mask)





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =self.prepare_config_and_inputs()
       snake_case  ,    snake_case  ,    snake_case   :							Optional[int]						  =config_and_inputs
       snake_case   :							Union[str, Any]						  ={'''input_ids''': input_ids, '''attention_mask''': attention_mask}
       return config, inputs_dict





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case  ,    snake_case   :							List[str]						  =self.prepare_config_and_inputs()

       snake_case   :							Any						  =floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
       snake_case   :							List[Any]						  =ids_tensor([self.batch_size, self.seq_length],    vocab_size=2     )

       return (
           config,
           input_ids,
           attention_mask,
           encoder_hidden_states,
           encoder_attention_mask,
       )





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    List[Any],    _snake_case    :    Optional[int],    _snake_case    :    Optional[int],    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							int						  =20
       snake_case   :							str						  =model_class_name(_snake_case     )

       snake_case   :							List[str]						  =model.init_cache(input_ids.shape[0],    _snake_case     )
       snake_case   :							Dict						  =jnp.ones((input_ids.shape[0], max_decoder_length),    dtype='''i4'''     )

       snake_case   :							Optional[int]						  =jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1     )[None, :],    (input_ids.shape[0], input_ids.shape[-1] - 1)     )
       snake_case   :							str						  =model(
           input_ids[:, :-1],    attention_mask=_snake_case,    past_key_values=_snake_case,    position_ids=_snake_case,    )

       snake_case   :							str						  =jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]],    dtype='''i4'''     )
       snake_case   :							int						  =model(
           input_ids[:, -1:],    attention_mask=_snake_case,    past_key_values=outputs_cache.past_key_values,    position_ids=_snake_case,    )

       snake_case   :							Dict						  =model(_snake_case     )

       snake_case   :							Any						  =np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
       self.parent.assertTrue(diff < 1E-3,    msg=f'''Max diff is {diff}'''     )






    def 					__snake_case       (    self    :    Any,    _snake_case    :    Tuple,    _snake_case    :    str,    _snake_case    :    Tuple,    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =20
       snake_case   :							Tuple						  =model_class_name(_snake_case     )

       snake_case   :							Tuple						  =jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])     )],    axis=-1,    )

       snake_case   :							Dict						  =model.init_cache(input_ids.shape[0],    _snake_case     )
       snake_case   :							List[str]						  =jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1     )[None, :],    (input_ids.shape[0], input_ids.shape[-1] - 1)     )

       snake_case   :							str						  =model(
           input_ids[:, :-1],    attention_mask=_snake_case,    past_key_values=_snake_case,    position_ids=_snake_case,    )
       snake_case   :							Dict						  =jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]],    dtype='''i4'''     )
       snake_case   :							Tuple						  =model(
           input_ids[:, -1:],    past_key_values=outputs_cache.past_key_values,    attention_mask=_snake_case,    position_ids=_snake_case,    )

       snake_case   :							Any						  =model(_snake_case,    attention_mask=_snake_case     )

       snake_case   :							int						  =np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
       self.parent.assertTrue(diff < 1E-3,    msg=f'''Max diff is {diff}'''     )





@require_flax
class 							lowerCAmelCase_		(	a_  ,      a_  ,      unittest.TestCase						):
    __UpperCAmelCase							=	(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
    __UpperCAmelCase							=	(FlaxGPTaLMHeadModel,) if is_flax_available() else ()





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							List[str]						  =FlaxGPTaModelTester(self     )





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       for model_class_name in self.all_model_classes:
          snake_case  ,    snake_case  ,    snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
          self.model_tester.check_use_cache_forward(_snake_case,    _snake_case,    _snake_case,    _snake_case     )





    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       for model_class_name in self.all_model_classes:
          snake_case  ,    snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs()
          self.model_tester.check_use_cache_forward_with_attn_mask(
              _snake_case,    _snake_case,    _snake_case,    _snake_case     )





    @slow
    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =GPTaTokenizer.from_pretrained('''gpt2''',    pad_token='''</s>''',    padding_side='''left'''     )
       snake_case   :							List[str]						  =tokenizer(['''Hello this is a long string''', '''Hey'''],    return_tensors='''np''',    padding=_snake_case,    truncation=_snake_case     )

       snake_case   :							Optional[int]						  =FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''     )
       snake_case   :							Tuple						  =False
       snake_case   :							int						  =model.config.eos_token_id

       snake_case   :							List[str]						  =jax.jit(model.generate     )

       snake_case   :							Optional[int]						  =jit_generate(inputs['''input_ids'''],    attention_mask=inputs['''attention_mask''']     ).sequences

       snake_case   :							str						  =tokenizer.batch_decode(_snake_case,    skip_special_tokens=_snake_case     )

       snake_case   :							Union[str, Any]						  =[
           '''Hello this is a long string of words. I\'m going to start with the first one.\n''',
           '''Hey, I\'m not sure if I\'m going to be able to do''',
       ]

       self.assertListEqual(_snake_case,    _snake_case     )





    @is_pt_flax_cross_test
    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Union[str, Any]						  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
          with self.subTest(model_class.__name__     ):
             # prepare inputs
             snake_case   :							Tuple						  =self._prepare_for_class(_snake_case,    _snake_case     )
             snake_case   :							Dict						  ={k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

             # load corresponding PyTorch class
             snake_case   :							Any						  =model_class.__name__[4:]  # Skip the "Flax" at the beginning
             snake_case   :							int						  =getattr(_snake_case,    _snake_case     )

             snake_case  ,    snake_case   :							str						  =pt_inputs['''input_ids'''].shape
             snake_case   :							List[str]						  =np.random.randint(0,    seq_length - 1,    size=(batch_size,)     )
             for batch_idx, start_index in enumerate(_snake_case     ):
                snake_case   :							Dict						  =0
                snake_case   :							Dict						  =1
                snake_case   :							Tuple						  =0
                snake_case   :							Tuple						  =1
             snake_case   :							Tuple						  =pt_model_class(_snake_case     ).eval()
             snake_case   :							Union[str, Any]						  =model_class(_snake_case,    dtype=jnp.floataa     )

             snake_case   :							str						  =convert_pytorch_state_dict_to_flax(pt_model.state_dict(),    _snake_case     )
             snake_case   :							List[Any]						  =fx_state

             with torch.no_grad():
                snake_case   :							Union[str, Any]						  =pt_model(**_snake_case     ).to_tuple()

             snake_case   :							Union[str, Any]						  =fx_model(**_snake_case     ).to_tuple()
             self.assertEqual(len(_snake_case     ),    len(_snake_case     ),    '''Output lengths differ between Flax and PyTorch'''     )
             for fx_output, pt_output in zip(_snake_case,    _snake_case     ):
                self.assert_almost_equals(fx_output[:, -1],    pt_output[:, -1].numpy(),    4E-2     )

             with tempfile.TemporaryDirectory() as tmpdirname:
                pt_model.save_pretrained(_snake_case     )
                snake_case   :							int						  =model_class.from_pretrained(_snake_case,    from_pt=_snake_case     )

             snake_case   :							Optional[Any]						  =fx_model_loaded(**_snake_case     ).to_tuple()
             self.assertEqual(
                 len(_snake_case     ),    len(_snake_case     ),    '''Output lengths differ between Flax and PyTorch'''     )
             for fx_output_loaded, pt_output in zip(_snake_case,    _snake_case     ):
                self.assert_almost_equals(fx_output_loaded[:, -1],    pt_output[:, -1].numpy(),    4E-2     )





    @is_pt_flax_cross_test
    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Union[str, Any]						  =self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
          with self.subTest(model_class.__name__     ):
             # prepare inputs
             snake_case   :							Optional[int]						  =self._prepare_for_class(_snake_case,    _snake_case     )
             snake_case   :							str						  ={k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

             # load corresponding PyTorch class
             snake_case   :							Optional[Any]						  =model_class.__name__[4:]  # Skip the "Flax" at the beginning
             snake_case   :							Union[str, Any]						  =getattr(_snake_case,    _snake_case     )

             snake_case   :							Optional[int]						  =pt_model_class(_snake_case     ).eval()
             snake_case   :							Tuple						  =model_class(_snake_case,    dtype=jnp.floataa     )

             snake_case   :							int						  =load_flax_weights_in_pytorch_model(_snake_case,    fx_model.params     )
             snake_case  ,    snake_case   :							Dict						  =pt_inputs['''input_ids'''].shape
             snake_case   :							List[Any]						  =np.random.randint(0,    seq_length - 1,    size=(batch_size,)     )
             for batch_idx, start_index in enumerate(_snake_case     ):
                snake_case   :							Optional[Any]						  =0
                snake_case   :							List[Any]						  =1
                snake_case   :							Tuple						  =0
                snake_case   :							Union[str, Any]						  =1

             # make sure weights are tied in PyTorch
             pt_model.tie_weights()

             with torch.no_grad():
                snake_case   :							List[Any]						  =pt_model(**_snake_case     ).to_tuple()

             snake_case   :							int						  =fx_model(**_snake_case     ).to_tuple()
             self.assertEqual(len(_snake_case     ),    len(_snake_case     ),    '''Output lengths differ between Flax and PyTorch'''     )
             for fx_output, pt_output in zip(_snake_case,    _snake_case     ):
                self.assert_almost_equals(fx_output[:, -1],    pt_output[:, -1].numpy(),    4E-2     )

             with tempfile.TemporaryDirectory() as tmpdirname:
                fx_model.save_pretrained(_snake_case     )
                snake_case   :							Tuple						  =pt_model_class.from_pretrained(_snake_case,    from_flax=_snake_case     )

             with torch.no_grad():
                snake_case   :							List[str]						  =pt_model_loaded(**_snake_case     ).to_tuple()

             self.assertEqual(
                 len(_snake_case     ),    len(_snake_case     ),    '''Output lengths differ between Flax and PyTorch'''     )
             for fx_output, pt_output in zip(_snake_case,    _snake_case     ):
                self.assert_almost_equals(fx_output[:, -1],    pt_output[:, -1].numpy(),    4E-2     )






    @slow
    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       for model_class_name in self.all_model_classes:
          snake_case   :							Optional[int]						  =model_class_name.from_pretrained('''gpt2''',    from_pt=_snake_case     )
          snake_case   :							Union[str, Any]						  =model(np.ones((1, 1)     )     )
          self.assertIsNotNone(_snake_case     )










'''simple docstring'''





import collections
import inspect
import unittest

from transformers import SwinConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel
      from transformers.models.swin.modeling_swin import SWIN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor





class 							lowerCAmelCase_		:





    def __init__(    self    :    Tuple,    _snake_case    :    Union[str, Any],    _snake_case    :    str=13,    _snake_case    :    Union[str, Any]=32,    _snake_case    :    List[str]=2,    _snake_case    :    Optional[int]=3,    _snake_case    :    str=16,    _snake_case    :    Any=[1, 2, 1],    _snake_case    :    Optional[int]=[2, 2, 4],    _snake_case    :    Optional[Any]=2,    _snake_case    :    Dict=2.0,    _snake_case    :    List[str]=True,    _snake_case    :    Union[str, Any]=0.0,    _snake_case    :    Any=0.0,    _snake_case    :    int=0.1,    _snake_case    :    Union[str, Any]="gelu",    _snake_case    :    List[str]=False,    _snake_case    :    Any=True,    _snake_case    :    Dict=0.02,    _snake_case    :    Dict=1E-5,    _snake_case    :    Any=True,    _snake_case    :    Tuple=None,    _snake_case    :    Dict=True,    _snake_case    :    Tuple=10,    _snake_case    :    List[str]=8,    _snake_case    :    Optional[Any]=["stage1", "stage2"],    _snake_case    :    Tuple=[1, 2],    ):



       '''simple docstring'''







       snake_case   :							List[str]						  =parent
       snake_case   :							Any						  =batch_size
       snake_case   :							List[str]						  =image_size
       snake_case   :							Tuple						  =patch_size
       snake_case   :							Tuple						  =num_channels
       snake_case   :							str						  =embed_dim
       snake_case   :							Optional[Any]						  =depths
       snake_case   :							str						  =num_heads
       snake_case   :							Tuple						  =window_size
       snake_case   :							Any						  =mlp_ratio
       snake_case   :							List[Any]						  =qkv_bias
       snake_case   :							Dict						  =hidden_dropout_prob
       snake_case   :							str						  =attention_probs_dropout_prob
       snake_case   :							List[str]						  =drop_path_rate
       snake_case   :							Tuple						  =hidden_act
       snake_case   :							Tuple						  =use_absolute_embeddings
       snake_case   :							List[Any]						  =patch_norm
       snake_case   :							Any						  =layer_norm_eps
       snake_case   :							Union[str, Any]						  =initializer_range
       snake_case   :							int						  =is_training
       snake_case   :							Dict						  =scope
       snake_case   :							Tuple						  =use_labels
       snake_case   :							Dict						  =type_sequence_label_size
       snake_case   :							Optional[Any]						  =encoder_stride
       snake_case   :							Union[str, Any]						  =out_features
       snake_case   :							Optional[Any]						  =out_indices





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

       snake_case   :							List[Any]						  =None
       if self.use_labels:
          snake_case   :							str						  =ids_tensor([self.batch_size],    self.type_sequence_label_size     )

       snake_case   :							str						  =self.get_config()

       return config, pixel_values, labels





    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       return SwinConfig(
           image_size=self.image_size,    patch_size=self.patch_size,    num_channels=self.num_channels,    embed_dim=self.embed_dim,    depths=self.depths,    num_heads=self.num_heads,    window_size=self.window_size,    mlp_ratio=self.mlp_ratio,    qkv_bias=self.qkv_bias,    hidden_dropout_prob=self.hidden_dropout_prob,    attention_probs_dropout_prob=self.attention_probs_dropout_prob,    drop_path_rate=self.drop_path_rate,    hidden_act=self.hidden_act,    use_absolute_embeddings=self.use_absolute_embeddings,    path_norm=self.patch_norm,    layer_norm_eps=self.layer_norm_eps,    initializer_range=self.initializer_range,    encoder_stride=self.encoder_stride,    out_features=self.out_features,    out_indices=self.out_indices,    )





    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    int,    _snake_case    :    List[str],    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Any						  =SwinModel(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Dict						  =model(_snake_case     )

       snake_case   :							List[Any]						  =((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths     ) - 1))
       snake_case   :							Dict						  =int(config.embed_dim * 2 ** (len(config.depths     ) - 1)     )

       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, expected_seq_len, expected_dim)     )





    def 					__snake_case       (    self    :    Dict,    _snake_case    :    List[Any],    _snake_case    :    Optional[Any],    _snake_case    :    Any     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =SwinBackbone(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Optional[int]						  =model(_snake_case     )

       # verify hidden states
       self.parent.assertEqual(len(result.feature_maps     ),    len(config.out_features     )     )
       self.parent.assertListEqual(list(result.feature_maps[0].shape     ),    [self.batch_size, model.channels[0], 16, 16]     )

       # verify channels
       self.parent.assertEqual(len(model.channels     ),    len(config.out_features     )     )

       # verify backbone works with out_features=None
       snake_case   :							int						  =None
       snake_case   :							List[str]						  =SwinBackbone(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Union[str, Any]						  =model(_snake_case     )

       # verify feature maps
       self.parent.assertEqual(len(result.feature_maps     ),    1     )
       self.parent.assertListEqual(list(result.feature_maps[0].shape     ),    [self.batch_size, model.channels[-1], 4, 4]     )

       # verify channels
       self.parent.assertEqual(len(model.channels     ),    1     )





    def 					__snake_case       (    self    :    str,    _snake_case    :    List[str],    _snake_case    :    int,    _snake_case    :    str     ):



       '''simple docstring'''







       snake_case   :							int						  =SwinForMaskedImageModeling(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Union[str, Any]						  =model(_snake_case     )
       self.parent.assertEqual(
           result.logits.shape,    (self.batch_size, self.num_channels, self.image_size, self.image_size)     )

       # test greyscale images
       snake_case   :							Any						  =1
       snake_case   :							List[Any]						  =SwinForMaskedImageModeling(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       snake_case   :							str						  =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
       snake_case   :							int						  =model(_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, 1, self.image_size, self.image_size)     )





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    str,    _snake_case    :    Union[str, Any],    _snake_case    :    Dict     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.type_sequence_label_size
       snake_case   :							Tuple						  =SwinForImageClassification(_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							str						  =model(_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.type_sequence_label_size)     )

       # test greyscale images
       snake_case   :							Union[str, Any]						  =1
       snake_case   :							List[str]						  =SwinForImageClassification(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       snake_case   :							Any						  =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
       snake_case   :							List[str]						  =model(_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.type_sequence_label_size)     )






    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Union[str, Any]						  =self.prepare_config_and_inputs()
       (
           (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    
       )   :							List[str]						  =config_and_inputs
       snake_case   :							int						  ={'''pixel_values''': pixel_values}
       return config, inputs_dict





@require_torch
class 							lowerCAmelCase_		(	a_  ,      a_  ,      unittest.TestCase						):
    __UpperCAmelCase							=	(
        (
            SwinModel,
            SwinBackbone,
            SwinForImageClassification,
            SwinForMaskedImageModeling,
        )
        if is_torch_available()
        else ()
    )
    __UpperCAmelCase							=	(
        {'feature-extraction': SwinModel, 'image-classification': SwinForImageClassification}
        if is_torch_available()
        else {}
    )
    __UpperCAmelCase							=	True

    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =SwinModelTester(self     )
       snake_case   :							int						  =ConfigTester(self,    config_class=_snake_case,    embed_dim=37     )





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       self.create_and_test_config_common_properties()
       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       return





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       snake_case   :							Dict						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_snake_case     )





    @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.'''     )
    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       pass





    def 					__snake_case       (    self    :    Optional[int]     ):



       '''simple docstring'''







       super().test_training_gradient_checkpointing()





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_backbone(*_snake_case     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							List[str]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_image_modeling(*_snake_case     )





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*_snake_case     )





    @unittest.skip(reason='''Swin does not use inputs_embeds'''     )
    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       pass





    @unittest.skip(reason='''Swin Transformer does not use feedforward chunking'''     )
    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       pass





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Dict						  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
          snake_case   :							Dict						  =model_class(_snake_case     )
          self.assertIsInstance(model.get_input_embeddings(),    (nn.Module)     )
          snake_case   :							str						  =model.get_output_embeddings()
          self.assertTrue(x is None or isinstance(_snake_case,    nn.Linear     )     )





    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Optional[int]						  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
          snake_case   :							Optional[int]						  =model_class(_snake_case     )
          snake_case   :							List[str]						  =inspect.signature(model.forward     )
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          snake_case   :							str						  =[*signature.parameters.keys()]

          snake_case   :							Union[str, Any]						  =['''pixel_values''']
          self.assertListEqual(arg_names[:1],    _snake_case     )





    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()
       snake_case   :							int						  =True

       for model_class in self.all_model_classes:
          snake_case   :							Optional[int]						  =True
          snake_case   :							Optional[int]						  =False
          snake_case   :							Optional[int]						  =True
          snake_case   :							Union[str, Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Tuple						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )
          snake_case   :							Optional[int]						  =outputs.attentions
          snake_case   :							List[str]						  =len(self.model_tester.depths     )
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          # check that output_attentions also work using config
          del inputs_dict["output_attentions"]
          snake_case   :							Tuple						  =True
          snake_case   :							int						  =config.window_size**2
          snake_case   :							List[Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Optional[Any]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )
          snake_case   :							Union[str, Any]						  =outputs.attentions
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          self.assertListEqual(
              list(attentions[0].shape[-3:]     ),    [self.model_tester.num_heads[0], window_size_squared, window_size_squared],    )
          snake_case   :							str						  =len(_snake_case     )

          # Check attention is always last and order is fine
          snake_case   :							Dict						  =True
          snake_case   :							Any						  =True
          snake_case   :							Optional[Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Optional[int]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )

          # also another +1 for reshaped_hidden_states
          snake_case   :							Union[str, Any]						  =1 if model_class.__name__ == '''SwinBackbone''' else 2
          self.assertEqual(out_len + added_hidden_states,    len(_snake_case     )     )

          snake_case   :							Tuple						  =outputs.attentions

          self.assertEqual(len(_snake_case     ),    _snake_case     )

          self.assertListEqual(
              list(self_attentions[0].shape[-3:]     ),    [self.model_tester.num_heads[0], window_size_squared, window_size_squared],    )





    def 					__snake_case       (    self    :    Any,    _snake_case    :    Tuple,    _snake_case    :    List[Any],    _snake_case    :    List[str],    _snake_case    :    Optional[int]     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =model_class(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       with torch.no_grad():
          snake_case   :							List[str]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )

       snake_case   :							int						  =outputs.hidden_states

       snake_case   :							Optional[int]						  =getattr(
           self.model_tester,    '''expected_num_hidden_layers''',    len(self.model_tester.depths     ) + 1     )
       self.assertEqual(len(_snake_case     ),    _snake_case     )

       # Swin has a different seq_length
       snake_case   :							Union[str, Any]						  =(
           config.patch_size
           if isinstance(config.patch_size,    collections.abc.Iterable     )
           else (config.patch_size, config.patch_size)
       )

       snake_case   :							Tuple						  =(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]     ),    [num_patches, self.model_tester.embed_dim],    )

       if not model_class.__name__ == "SwinBackbone":
          snake_case   :							List[str]						  =outputs.reshaped_hidden_states
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          snake_case  ,    snake_case  ,    snake_case  ,    snake_case   :							int						  =reshaped_hidden_states[0].shape
          snake_case   :							str						  =(
              reshaped_hidden_states[0].view(_snake_case,    _snake_case,    height * width     ).permute(0,    2,    1     )
          )
          self.assertListEqual(
              list(reshaped_hidden_states.shape[-2:]     ),    [num_patches, self.model_tester.embed_dim],    )





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()

       snake_case   :							Optional[Any]						  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size,    collections.abc.Iterable     )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )

       for model_class in self.all_model_classes:
          snake_case   :							int						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    _snake_case     )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          snake_case   :							str						  =True

          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    _snake_case     )





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()
       snake_case   :							Union[str, Any]						  =3

       snake_case   :							Optional[Any]						  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size,    collections.abc.Iterable     )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )
       snake_case   :							Optional[int]						  =(
           config.patch_size
           if isinstance(config.patch_size,    collections.abc.Iterable     )
           else (config.patch_size, config.patch_size)
       )

       snake_case   :							Any						  =image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
       snake_case   :							Union[str, Any]						  =image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

       for model_class in self.all_model_classes:
          snake_case   :							List[str]						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    (padded_height, padded_width)     )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          snake_case   :							Union[str, Any]						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    (padded_height, padded_width)     )





    @slow
    def 					__snake_case       (    self    :    Optional[int]     ):



       '''simple docstring'''







       for model_name in SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          snake_case   :							str						  =SwinModel.from_pretrained(_snake_case     )
          self.assertIsNotNone(_snake_case     )






    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Tuple						  =self.model_tester.prepare_config_and_inputs_for_common()

       snake_case   :							Dict						  =_config_zero_init(_snake_case     )
       for model_class in self.all_model_classes:
          snake_case   :							List[str]						  =model_class(config=_snake_case     )
          for name, param in model.named_parameters():
             if "embeddings" not in name and param.requires_grad:
                self.assertIn(
                    ((param.data.mean() * 1E9).round() / 1E9).item(),    [0.0, 1.0],    msg=f'''Parameter {name} of model {model_class} seems not properly initialized''',    )





@require_vision
@require_torch
class 							lowerCAmelCase_		(	unittest.TestCase						):





    @cached_property
    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       return (
           AutoImageProcessor.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''     )
           if is_vision_available()
           else None
       )






    @slow
    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case   :							Dict						  =SwinForImageClassification.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''     ).to(_snake_case     )
       snake_case   :							Union[str, Any]						  =self.default_image_processor

       snake_case   :							Union[str, Any]						  =Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
       snake_case   :							int						  =image_processor(images=_snake_case,    return_tensors='''pt'''     ).to(_snake_case     )

       # forward pass
       with torch.no_grad():
          snake_case   :							Any						  =model(**_snake_case     )

       # verify the logits
       snake_case   :							Dict						  =torch.Size((1, 1_000)     )
       self.assertEqual(outputs.logits.shape,    _snake_case     )
       snake_case   :							int						  =torch.tensor([-0.0948, -0.6454, -0.0921]     ).to(_snake_case     )
       self.assertTrue(torch.allclose(outputs.logits[0, :3],    _snake_case,    atol=1E-4     )     )





@require_torch
class 							lowerCAmelCase_		(	unittest.TestCase  ,      a_						):
    __UpperCAmelCase							=	(SwinBackbone,) if is_torch_available() else ()
    __UpperCAmelCase							=	SwinConfig





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =SwinModelTester(self     )





