



'''simple docstring'''





import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          FunnelBaseModel,
          FunnelForMaskedLM,
          FunnelForMultipleChoice,
          FunnelForPreTraining,
          FunnelForQuestionAnswering,
          FunnelForSequenceClassification,
          FunnelForTokenClassification,
          FunnelModel,
      )





class 							lowerCAmelCase_		:





    def __init__(    self    :    Optional[Any],    _snake_case    :    Optional[int],    _snake_case    :    Tuple=13,    _snake_case    :    Any=7,    _snake_case    :    List[Any]=True,    _snake_case    :    Optional[int]=True,    _snake_case    :    List[str]=True,    _snake_case    :    Dict=True,    _snake_case    :    Dict=99,    _snake_case    :    int=[1, 1, 2],    _snake_case    :    Optional[int]=1,    _snake_case    :    Optional[int]=32,    _snake_case    :    List[str]=4,    _snake_case    :    List[Any]=8,    _snake_case    :    List[Any]=37,    _snake_case    :    Any="gelu_new",    _snake_case    :    Optional[Any]=0.1,    _snake_case    :    Tuple=0.1,    _snake_case    :    Optional[int]=0.0,    _snake_case    :    Any=512,    _snake_case    :    Union[str, Any]=3,    _snake_case    :    Tuple=0.02,    _snake_case    :    Union[str, Any]=3,    _snake_case    :    Optional[int]=4,    _snake_case    :    Union[str, Any]=None,    _snake_case    :    List[str]=False,    ):



           '''simple docstring'''







           snake_case   :							Any						  =parent
           snake_case   :							Dict						  =batch_size
           snake_case   :							Optional[Any]						  =seq_length
           snake_case   :							Union[str, Any]						  =is_training
           snake_case   :							Any						  =use_input_mask
           snake_case   :							Dict						  =use_token_type_ids
           snake_case   :							Optional[int]						  =use_labels
           snake_case   :							Optional[Any]						  =vocab_size
           snake_case   :							Union[str, Any]						  =block_sizes
           snake_case   :							Dict						  =num_decoder_layers
           snake_case   :							Tuple						  =d_model
           snake_case   :							List[str]						  =n_head
           snake_case   :							List[Any]						  =d_head
           snake_case   :							Optional[int]						  =d_inner
           snake_case   :							Optional[int]						  =hidden_act
           snake_case   :							List[Any]						  =hidden_dropout
           snake_case   :							Any						  =attention_dropout
           snake_case   :							Union[str, Any]						  =activation_dropout
           snake_case   :							Union[str, Any]						  =max_position_embeddings
           snake_case   :							Union[str, Any]						  =type_vocab_size
           snake_case   :							Optional[Any]						  =2
           snake_case   :							Union[str, Any]						  =num_labels
           snake_case   :							Dict						  =num_choices
           snake_case   :							List[str]						  =scope
           snake_case   :							str						  =initializer_std

           # Used in the tests to check the size of the first attention layer
           snake_case   :							Optional[Any]						  =n_head
           # Used in the tests to check the size of the first hidden state
           snake_case   :							List[Any]						  =self.d_model
           # Used in the tests to check the number of output hidden states/attentions
           snake_case   :							int						  =sum(self.block_sizes     ) + (0 if base else self.num_decoder_layers)
           # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
           # the last hidden state of the first block (which is the first hidden state of the decoder).
           if not base:
                  snake_case   :							str						  =self.num_hidden_layers + 2





    def 					__snake_case       (    self    :    Union[str, Any]     ):



           '''simple docstring'''







           snake_case   :							Tuple						  =ids_tensor([self.batch_size, self.seq_length],    self.vocab_size     )

           snake_case   :							Tuple						  =None
           if self.use_input_mask:
                  snake_case   :							int						  =ids_tensor([self.batch_size, self.seq_length],    vocab_size=2     )

           snake_case   :							Union[str, Any]						  =None
           if self.use_token_type_ids:
                  snake_case   :							Any						  =ids_tensor([self.batch_size, self.seq_length],    self.type_vocab_size     )

           snake_case   :							List[str]						  =None
           snake_case   :							str						  =None
           snake_case   :							Any						  =None
           if self.use_labels:
                  snake_case   :							List[str]						  =ids_tensor([self.batch_size],    self.type_sequence_label_size     )
                  snake_case   :							Optional[Any]						  =ids_tensor([self.batch_size, self.seq_length],    self.num_labels     )
                  snake_case   :							Tuple						  =ids_tensor([self.batch_size],    self.num_choices     )
                  snake_case   :							Tuple						  =ids_tensor([self.batch_size, self.seq_length],    1     )

           snake_case   :							str						  =self.get_config()

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               fake_token_labels,
           )





    def 					__snake_case       (    self    :    Tuple     ):



           '''simple docstring'''







           return FunnelConfig(
               vocab_size=self.vocab_size,    block_sizes=self.block_sizes,    num_decoder_layers=self.num_decoder_layers,    d_model=self.d_model,    n_head=self.n_head,    d_head=self.d_head,    d_inner=self.d_inner,    hidden_act=self.hidden_act,    hidden_dropout=self.hidden_dropout,    attention_dropout=self.attention_dropout,    activation_dropout=self.activation_dropout,    max_position_embeddings=self.max_position_embeddings,    type_vocab_size=self.type_vocab_size,    initializer_std=self.initializer_std,    )





    def 					__snake_case       (    self    :    Union[str, Any],    _snake_case    :    List[Any],    _snake_case    :    Optional[Any],    _snake_case    :    List[str],    _snake_case    :    List[Any],    _snake_case    :    Any,    _snake_case    :    str,    _snake_case    :    Any,    _snake_case    :    List[str],    ):



           '''simple docstring'''







           snake_case   :							Union[str, Any]						  =FunnelModel(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							int						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase     )
           snake_case   :							Optional[int]						  =model(__lowercase,    token_type_ids=__lowercase     )
           snake_case   :							str						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )

           snake_case   :							Optional[int]						  =False
           snake_case   :							Optional[int]						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )

           snake_case   :							Optional[Any]						  =False
           snake_case   :							Union[str, Any]						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )





    def 					__snake_case       (    self    :    Union[str, Any],    _snake_case    :    List[str],    _snake_case    :    str,    _snake_case    :    Optional[Any],    _snake_case    :    Any,    _snake_case    :    int,    _snake_case    :    List[str],    _snake_case    :    str,    _snake_case    :    str,    ):



           '''simple docstring'''







           snake_case   :							Optional[int]						  =FunnelBaseModel(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Union[str, Any]						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase     )
           snake_case   :							Optional[int]						  =model(__lowercase,    token_type_ids=__lowercase     )
           snake_case   :							Optional[Any]						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 2, self.d_model)     )

           snake_case   :							Optional[int]						  =False
           snake_case   :							Optional[Any]						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 3, self.d_model)     )

           snake_case   :							Any						  =False
           snake_case   :							str						  =model(__lowercase     )
           self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 2, self.d_model)     )





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    Any,    _snake_case    :    List[str],    _snake_case    :    str,    _snake_case    :    Union[str, Any],    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    ):



           '''simple docstring'''







           snake_case   :							List[str]						  =self.num_labels
           snake_case   :							Any						  =FunnelForPreTraining(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Optional[int]						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    labels=__lowercase     )
           self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length)     )





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    str,    _snake_case    :    List[str],    _snake_case    :    Any,    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    _snake_case    :    int,    _snake_case    :    Dict,    _snake_case    :    Dict,    ):



           '''simple docstring'''







           snake_case   :							Any						  =FunnelForMaskedLM(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Optional[int]						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    labels=__lowercase     )
           self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.vocab_size)     )





    def 					__snake_case       (    self    :    Any,    _snake_case    :    Dict,    _snake_case    :    List[Any],    _snake_case    :    Any,    _snake_case    :    Optional[Any],    _snake_case    :    Optional[Any],    _snake_case    :    Dict,    _snake_case    :    Any,    _snake_case    :    List[str],    ):



           '''simple docstring'''







           snake_case   :							str						  =self.num_labels
           snake_case   :							int						  =FunnelForSequenceClassification(__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Optional[int]						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    labels=__lowercase     )
           self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.num_labels)     )





    def 					__snake_case       (    self    :    Union[str, Any],    _snake_case    :    str,    _snake_case    :    Dict,    _snake_case    :    Dict,    _snake_case    :    Dict,    _snake_case    :    List[str],    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    _snake_case    :    Dict,    ):



           '''simple docstring'''







           snake_case   :							Optional[Any]						  =self.num_choices
           snake_case   :							List[Any]						  =FunnelForMultipleChoice(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Tuple						  =input_ids.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
           snake_case   :							Optional[int]						  =token_type_ids.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
           snake_case   :							Any						  =input_mask.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
           snake_case   :							Union[str, Any]						  =model(
               __lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    labels=__lowercase,    )
           self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.num_choices)     )





    def 					__snake_case       (    self    :    Tuple,    _snake_case    :    Optional[int],    _snake_case    :    Union[str, Any],    _snake_case    :    int,    _snake_case    :    Dict,    _snake_case    :    List[str],    _snake_case    :    Tuple,    _snake_case    :    List[str],    _snake_case    :    List[Any],    ):



           '''simple docstring'''







           snake_case   :							Optional[Any]						  =self.num_labels
           snake_case   :							List[Any]						  =FunnelForTokenClassification(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Tuple						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    labels=__lowercase     )
           self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.num_labels)     )





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    Any,    _snake_case    :    List[Any],    _snake_case    :    List[Any],    _snake_case    :    Optional[Any],    _snake_case    :    Dict,    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[int],    _snake_case    :    Optional[int],    ):



           '''simple docstring'''







           snake_case   :							str						  =FunnelForQuestionAnswering(config=__lowercase     )
           model.to(__lowercase     )
           model.eval()
           snake_case   :							Any						  =model(
               __lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase,    start_positions=__lowercase,    end_positions=__lowercase,    )
           self.parent.assertEqual(result.start_logits.shape,    (self.batch_size, self.seq_length)     )
           self.parent.assertEqual(result.end_logits.shape,    (self.batch_size, self.seq_length)     )






    def 					__snake_case       (    self    :    int     ):



           '''simple docstring'''







           snake_case   :							str						  =self.prepare_config_and_inputs()
           (
               (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    (
               snake_case				
           )  ,    				
           )   :							int						  =config_and_inputs
           snake_case   :							str						  ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
           return config, inputs_dict





@require_torch
class 							lowerCAmelCase_		(	UpperCAmelCase__  ,      UpperCAmelCase__  ,      unittest.TestCase						):
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	(
        (
            FunnelModel,
            FunnelForMaskedLM,
            FunnelForPreTraining,
            FunnelForQuestionAnswering,
            FunnelForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    __UpperCAmelCase							=	(
        {
            'feature-extraction': (FunnelBaseModel, FunnelModel),
            'fill-mask': FunnelForMaskedLM,
            'question-answering': FunnelForQuestionAnswering,
            'text-classification': FunnelForSequenceClassification,
            'token-classification': FunnelForTokenClassification,
            'zero-shot': FunnelForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )





    def 					__snake_case       (    self    :    Dict,    _snake_case    :    Any,    _snake_case    :    Optional[Any],    _snake_case    :    Optional[int]=False     ):



           '''simple docstring'''







           snake_case   :							Any						  =super()._prepare_for_class(__lowercase,    __lowercase,    return_labels=__lowercase     )

           if return_labels:
                  if model_class in get_values(__lowercase     ):
                         snake_case   :							Optional[int]						  =torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length),    dtype=torch.long,    device=__lowercase     )
           return inputs_dict





    def 					__snake_case       (    self    :    int     ):



           '''simple docstring'''







           snake_case   :							Any						  =FunnelModelTester(self     )
           snake_case   :							Optional[int]						  =ConfigTester(self,    config_class=__lowercase     )





    def 					__snake_case       (    self    :    Optional[int]     ):



           '''simple docstring'''







           self.config_tester.run_common_tests()





    def 					__snake_case       (    self    :    List[Any]     ):



           '''simple docstring'''







           snake_case   :							Dict						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*__lowercase     )





    def 					__snake_case       (    self    :    List[str]     ):



           '''simple docstring'''







           snake_case   :							Optional[Any]						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*__lowercase     )





    def 					__snake_case       (    self    :    Tuple     ):



           '''simple docstring'''







           snake_case   :							str						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*__lowercase     )





    def 					__snake_case       (    self    :    Dict     ):



           '''simple docstring'''







           snake_case   :							List[Any]						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*__lowercase     )





    def 					__snake_case       (    self    :    List[str]     ):



           '''simple docstring'''







           snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*__lowercase     )






    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    Tuple     ):



           '''simple docstring'''







           if hasattr(__lowercase,    '''weight'''     ) and module.weight is not None:
                  module.weight.data.fill_(3     )
           if hasattr(__lowercase,    '''bias'''     ) and module.bias is not None:
                  module.bias.data.fill_(3     )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(__lowercase,    __lowercase     ) and getattr(__lowercase,    __lowercase     ) is not None:
                         snake_case   :							Optional[int]						  =getattr(__lowercase,    __lowercase     )
                         weight.data.fill_(3     )





@require_torch
class 							lowerCAmelCase_		(	UpperCAmelCase__  ,      unittest.TestCase						):
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	(
        (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
    )





    def 					__snake_case       (    self    :    Dict     ):



           '''simple docstring'''







           snake_case   :							List[str]						  =FunnelModelTester(self,    base=__lowercase     )
           snake_case   :							List[str]						  =ConfigTester(self,    config_class=__lowercase     )





    def 					__snake_case       (    self    :    str     ):



           '''simple docstring'''







           self.config_tester.run_common_tests()





    def 					__snake_case       (    self    :    Optional[Any]     ):



           '''simple docstring'''







           snake_case   :							str						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_base_model(*__lowercase     )





    def 					__snake_case       (    self    :    List[Any]     ):



           '''simple docstring'''







           snake_case   :							int						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*__lowercase     )





    def 					__snake_case       (    self    :    int     ):



           '''simple docstring'''







           snake_case   :							Optional[int]						  =self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*__lowercase     )





    def 					__snake_case       (    self    :    Dict     ):



           '''simple docstring'''







           snake_case  ,    snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs_for_common()
           snake_case   :							Union[str, Any]						  =True

           for model_class in self.all_model_classes:
                  if model_class.__name__ == "FunnelBaseModel":
                         continue
                  snake_case   :							Optional[Any]						  =model_class(__lowercase     )
                  model.to(__lowercase     )
                  model.train()
                  snake_case   :							Any						  =self._prepare_for_class(__lowercase,    __lowercase,    return_labels=__lowercase     )
                  snake_case   :							Optional[Any]						  =model(**__lowercase     ).loss
                  loss.backward()






    def 					__snake_case       (    self    :    str,    _snake_case    :    Optional[Any]     ):



           '''simple docstring'''







           if hasattr(__lowercase,    '''weight'''     ) and module.weight is not None:
                  module.weight.data.fill_(3     )
           if hasattr(__lowercase,    '''bias'''     ) and module.bias is not None:
                  module.bias.data.fill_(3     )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(__lowercase,    __lowercase     ) and getattr(__lowercase,    __lowercase     ) is not None:
                         snake_case   :							List[Any]						  =getattr(__lowercase,    __lowercase     )
                         weight.data.fill_(3     )





@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase_		(	unittest.TestCase						):





    def 					__snake_case       (    self    :    List[Any]     ):



           '''simple docstring'''







           snake_case   :							Tuple						  =13
           snake_case   :							str						  =7
           snake_case   :							Optional[int]						  =torch.arange(0,    batch_size * sequence_length     ).long().reshape(__lowercase,    __lowercase     )
           snake_case   :							Any						  =[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
           snake_case   :							List[str]						  =torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]     )

           snake_case   :							str						  =FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''     )
           snake_case   :							Dict						  =model(__lowercase,    token_type_ids=__lowercase     )[0].abs()

           snake_case   :							List[str]						  =torch.tensor(2344.8352     )
           snake_case   :							Dict						  =torch.tensor(0.8052     )
           self.assertTrue(torch.allclose(output.sum(),    __lowercase,    atol=1E-4     )     )
           self.assertTrue(torch.allclose(output.mean(),    __lowercase,    atol=1E-4     )     )

           snake_case   :							Tuple						  =torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]     )
           snake_case   :							str						  =model(__lowercase,    attention_mask=__lowercase,    token_type_ids=__lowercase     )[0].abs()

           snake_case   :							Any						  =torch.tensor(2343.8425     )
           snake_case   :							Optional[int]						  =torch.tensor(0.8049     )
           self.assertTrue(torch.allclose(output.sum(),    __lowercase,    atol=1E-4     )     )
           self.assertTrue(torch.allclose(output.mean(),    __lowercase,    atol=1E-4     )     )






    @slow
    def 					__snake_case       (    self    :    str     ):



           '''simple docstring'''







           snake_case   :							List[Any]						  =FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''     )
           snake_case   :							List[str]						  =FunnelModel.from_pretrained('''huggingface/funnel-small'''     )
           snake_case   :							Dict						  =tokenizer('''Hello! I am the Funnel Transformer model.''',    return_tensors='''pt'''     )
           snake_case   :							Union[str, Any]						  =model(**__lowercase     )[0]

           snake_case   :							Tuple						  =torch.tensor(235.7246     )
           snake_case   :							Union[str, Any]						  =torch.tensor(0.0256     )
           self.assertTrue(torch.allclose(output.sum(),    __lowercase,    atol=1E-4     )     )
           self.assertTrue(torch.allclose(output.mean(),    __lowercase,    atol=1E-4     )     )










'''simple docstring'''





import collections
import inspect
import unittest

from transformers import SwinConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel
      from transformers.models.swin.modeling_swin import SWIN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor





class 							lowerCAmelCase_		:





    def __init__(    self    :    Tuple,    _snake_case    :    Union[str, Any],    _snake_case    :    str=13,    _snake_case    :    Union[str, Any]=32,    _snake_case    :    List[str]=2,    _snake_case    :    Optional[int]=3,    _snake_case    :    str=16,    _snake_case    :    Any=[1, 2, 1],    _snake_case    :    Optional[int]=[2, 2, 4],    _snake_case    :    Optional[Any]=2,    _snake_case    :    Dict=2.0,    _snake_case    :    List[str]=True,    _snake_case    :    Union[str, Any]=0.0,    _snake_case    :    Any=0.0,    _snake_case    :    int=0.1,    _snake_case    :    Union[str, Any]="gelu",    _snake_case    :    List[str]=False,    _snake_case    :    Any=True,    _snake_case    :    Dict=0.02,    _snake_case    :    Dict=1E-5,    _snake_case    :    Any=True,    _snake_case    :    Tuple=None,    _snake_case    :    Dict=True,    _snake_case    :    Tuple=10,    _snake_case    :    List[str]=8,    _snake_case    :    Optional[Any]=["stage1", "stage2"],    _snake_case    :    Tuple=[1, 2],    ):



       '''simple docstring'''







       snake_case   :							List[str]						  =parent
       snake_case   :							Any						  =batch_size
       snake_case   :							List[str]						  =image_size
       snake_case   :							Tuple						  =patch_size
       snake_case   :							Tuple						  =num_channels
       snake_case   :							str						  =embed_dim
       snake_case   :							Optional[Any]						  =depths
       snake_case   :							str						  =num_heads
       snake_case   :							Tuple						  =window_size
       snake_case   :							Any						  =mlp_ratio
       snake_case   :							List[Any]						  =qkv_bias
       snake_case   :							Dict						  =hidden_dropout_prob
       snake_case   :							str						  =attention_probs_dropout_prob
       snake_case   :							List[str]						  =drop_path_rate
       snake_case   :							Tuple						  =hidden_act
       snake_case   :							Tuple						  =use_absolute_embeddings
       snake_case   :							List[Any]						  =patch_norm
       snake_case   :							Any						  =layer_norm_eps
       snake_case   :							Union[str, Any]						  =initializer_range
       snake_case   :							int						  =is_training
       snake_case   :							Dict						  =scope
       snake_case   :							Tuple						  =use_labels
       snake_case   :							Dict						  =type_sequence_label_size
       snake_case   :							Optional[Any]						  =encoder_stride
       snake_case   :							Union[str, Any]						  =out_features
       snake_case   :							Optional[Any]						  =out_indices





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

       snake_case   :							List[Any]						  =None
       if self.use_labels:
          snake_case   :							str						  =ids_tensor([self.batch_size],    self.type_sequence_label_size     )

       snake_case   :							str						  =self.get_config()

       return config, pixel_values, labels





    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       return SwinConfig(
           image_size=self.image_size,    patch_size=self.patch_size,    num_channels=self.num_channels,    embed_dim=self.embed_dim,    depths=self.depths,    num_heads=self.num_heads,    window_size=self.window_size,    mlp_ratio=self.mlp_ratio,    qkv_bias=self.qkv_bias,    hidden_dropout_prob=self.hidden_dropout_prob,    attention_probs_dropout_prob=self.attention_probs_dropout_prob,    drop_path_rate=self.drop_path_rate,    hidden_act=self.hidden_act,    use_absolute_embeddings=self.use_absolute_embeddings,    path_norm=self.patch_norm,    layer_norm_eps=self.layer_norm_eps,    initializer_range=self.initializer_range,    encoder_stride=self.encoder_stride,    out_features=self.out_features,    out_indices=self.out_indices,    )





    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    int,    _snake_case    :    List[str],    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Any						  =SwinModel(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Dict						  =model(_snake_case     )

       snake_case   :							List[Any]						  =((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths     ) - 1))
       snake_case   :							Dict						  =int(config.embed_dim * 2 ** (len(config.depths     ) - 1)     )

       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, expected_seq_len, expected_dim)     )





    def 					__snake_case       (    self    :    Dict,    _snake_case    :    List[Any],    _snake_case    :    Optional[Any],    _snake_case    :    Any     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =SwinBackbone(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Optional[int]						  =model(_snake_case     )

       # verify hidden states
       self.parent.assertEqual(len(result.feature_maps     ),    len(config.out_features     )     )
       self.parent.assertListEqual(list(result.feature_maps[0].shape     ),    [self.batch_size, model.channels[0], 16, 16]     )

       # verify channels
       self.parent.assertEqual(len(model.channels     ),    len(config.out_features     )     )

       # verify backbone works with out_features=None
       snake_case   :							int						  =None
       snake_case   :							List[str]						  =SwinBackbone(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Union[str, Any]						  =model(_snake_case     )

       # verify feature maps
       self.parent.assertEqual(len(result.feature_maps     ),    1     )
       self.parent.assertListEqual(list(result.feature_maps[0].shape     ),    [self.batch_size, model.channels[-1], 4, 4]     )

       # verify channels
       self.parent.assertEqual(len(model.channels     ),    1     )





    def 					__snake_case       (    self    :    str,    _snake_case    :    List[str],    _snake_case    :    int,    _snake_case    :    str     ):



       '''simple docstring'''







       snake_case   :							int						  =SwinForMaskedImageModeling(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Union[str, Any]						  =model(_snake_case     )
       self.parent.assertEqual(
           result.logits.shape,    (self.batch_size, self.num_channels, self.image_size, self.image_size)     )

       # test greyscale images
       snake_case   :							Any						  =1
       snake_case   :							List[Any]						  =SwinForMaskedImageModeling(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       snake_case   :							str						  =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
       snake_case   :							int						  =model(_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, 1, self.image_size, self.image_size)     )





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    str,    _snake_case    :    Union[str, Any],    _snake_case    :    Dict     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.type_sequence_label_size
       snake_case   :							Tuple						  =SwinForImageClassification(_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							str						  =model(_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.type_sequence_label_size)     )

       # test greyscale images
       snake_case   :							Union[str, Any]						  =1
       snake_case   :							List[str]						  =SwinForImageClassification(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       snake_case   :							Any						  =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
       snake_case   :							List[str]						  =model(_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.type_sequence_label_size)     )






    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Union[str, Any]						  =self.prepare_config_and_inputs()
       (
           (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    
       )   :							List[str]						  =config_and_inputs
       snake_case   :							int						  ={'''pixel_values''': pixel_values}
       return config, inputs_dict





@require_torch
class 							lowerCAmelCase_		(	a_  ,      a_  ,      unittest.TestCase						):
    __UpperCAmelCase							=	(
        (
            SwinModel,
            SwinBackbone,
            SwinForImageClassification,
            SwinForMaskedImageModeling,
        )
        if is_torch_available()
        else ()
    )
    __UpperCAmelCase							=	(
        {'feature-extraction': SwinModel, 'image-classification': SwinForImageClassification}
        if is_torch_available()
        else {}
    )
    __UpperCAmelCase							=	True

    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =SwinModelTester(self     )
       snake_case   :							int						  =ConfigTester(self,    config_class=_snake_case,    embed_dim=37     )





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       self.create_and_test_config_common_properties()
       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       return





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       snake_case   :							Dict						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_snake_case     )





    @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.'''     )
    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       pass





    def 					__snake_case       (    self    :    Optional[int]     ):



       '''simple docstring'''







       super().test_training_gradient_checkpointing()





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_backbone(*_snake_case     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							List[str]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_image_modeling(*_snake_case     )





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*_snake_case     )





    @unittest.skip(reason='''Swin does not use inputs_embeds'''     )
    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       pass





    @unittest.skip(reason='''Swin Transformer does not use feedforward chunking'''     )
    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       pass





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Dict						  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
          snake_case   :							Dict						  =model_class(_snake_case     )
          self.assertIsInstance(model.get_input_embeddings(),    (nn.Module)     )
          snake_case   :							str						  =model.get_output_embeddings()
          self.assertTrue(x is None or isinstance(_snake_case,    nn.Linear     )     )





    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Optional[int]						  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
          snake_case   :							Optional[int]						  =model_class(_snake_case     )
          snake_case   :							List[str]						  =inspect.signature(model.forward     )
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          snake_case   :							str						  =[*signature.parameters.keys()]

          snake_case   :							Union[str, Any]						  =['''pixel_values''']
          self.assertListEqual(arg_names[:1],    _snake_case     )





    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()
       snake_case   :							int						  =True

       for model_class in self.all_model_classes:
          snake_case   :							Optional[int]						  =True
          snake_case   :							Optional[int]						  =False
          snake_case   :							Optional[int]						  =True
          snake_case   :							Union[str, Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Tuple						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )
          snake_case   :							Optional[int]						  =outputs.attentions
          snake_case   :							List[str]						  =len(self.model_tester.depths     )
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          # check that output_attentions also work using config
          del inputs_dict["output_attentions"]
          snake_case   :							Tuple						  =True
          snake_case   :							int						  =config.window_size**2
          snake_case   :							List[Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Optional[Any]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )
          snake_case   :							Union[str, Any]						  =outputs.attentions
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          self.assertListEqual(
              list(attentions[0].shape[-3:]     ),    [self.model_tester.num_heads[0], window_size_squared, window_size_squared],    )
          snake_case   :							str						  =len(_snake_case     )

          # Check attention is always last and order is fine
          snake_case   :							Dict						  =True
          snake_case   :							Any						  =True
          snake_case   :							Optional[Any]						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.eval()
          with torch.no_grad():
             snake_case   :							Optional[int]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )

          # also another +1 for reshaped_hidden_states
          snake_case   :							Union[str, Any]						  =1 if model_class.__name__ == '''SwinBackbone''' else 2
          self.assertEqual(out_len + added_hidden_states,    len(_snake_case     )     )

          snake_case   :							Tuple						  =outputs.attentions

          self.assertEqual(len(_snake_case     ),    _snake_case     )

          self.assertListEqual(
              list(self_attentions[0].shape[-3:]     ),    [self.model_tester.num_heads[0], window_size_squared, window_size_squared],    )





    def 					__snake_case       (    self    :    Any,    _snake_case    :    Tuple,    _snake_case    :    List[Any],    _snake_case    :    List[str],    _snake_case    :    Optional[int]     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =model_class(_snake_case     )
       model.to(_snake_case     )
       model.eval()

       with torch.no_grad():
          snake_case   :							List[str]						  =model(**self._prepare_for_class(_snake_case,    _snake_case     )     )

       snake_case   :							int						  =outputs.hidden_states

       snake_case   :							Optional[int]						  =getattr(
           self.model_tester,    '''expected_num_hidden_layers''',    len(self.model_tester.depths     ) + 1     )
       self.assertEqual(len(_snake_case     ),    _snake_case     )

       # Swin has a different seq_length
       snake_case   :							Union[str, Any]						  =(
           config.patch_size
           if isinstance(config.patch_size,    collections.abc.Iterable     )
           else (config.patch_size, config.patch_size)
       )

       snake_case   :							Tuple						  =(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]     ),    [num_patches, self.model_tester.embed_dim],    )

       if not model_class.__name__ == "SwinBackbone":
          snake_case   :							List[str]						  =outputs.reshaped_hidden_states
          self.assertEqual(len(_snake_case     ),    _snake_case     )

          snake_case  ,    snake_case  ,    snake_case  ,    snake_case   :							int						  =reshaped_hidden_states[0].shape
          snake_case   :							str						  =(
              reshaped_hidden_states[0].view(_snake_case,    _snake_case,    height * width     ).permute(0,    2,    1     )
          )
          self.assertListEqual(
              list(reshaped_hidden_states.shape[-2:]     ),    [num_patches, self.model_tester.embed_dim],    )





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()

       snake_case   :							Optional[Any]						  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size,    collections.abc.Iterable     )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )

       for model_class in self.all_model_classes:
          snake_case   :							int						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    _snake_case     )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          snake_case   :							str						  =True

          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    _snake_case     )





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							str						  =self.model_tester.prepare_config_and_inputs_for_common()
       snake_case   :							Union[str, Any]						  =3

       snake_case   :							Optional[Any]						  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size,    collections.abc.Iterable     )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )
       snake_case   :							Optional[int]						  =(
           config.patch_size
           if isinstance(config.patch_size,    collections.abc.Iterable     )
           else (config.patch_size, config.patch_size)
       )

       snake_case   :							Any						  =image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
       snake_case   :							Union[str, Any]						  =image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

       for model_class in self.all_model_classes:
          snake_case   :							List[str]						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    (padded_height, padded_width)     )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          snake_case   :							Union[str, Any]						  =True
          self.check_hidden_states_output(_snake_case,    _snake_case,    _snake_case,    (padded_height, padded_width)     )





    @slow
    def 					__snake_case       (    self    :    Optional[int]     ):



       '''simple docstring'''







       for model_name in SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          snake_case   :							str						  =SwinModel.from_pretrained(_snake_case     )
          self.assertIsNotNone(_snake_case     )






    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Tuple						  =self.model_tester.prepare_config_and_inputs_for_common()

       snake_case   :							Dict						  =_config_zero_init(_snake_case     )
       for model_class in self.all_model_classes:
          snake_case   :							List[str]						  =model_class(config=_snake_case     )
          for name, param in model.named_parameters():
             if "embeddings" not in name and param.requires_grad:
                self.assertIn(
                    ((param.data.mean() * 1E9).round() / 1E9).item(),    [0.0, 1.0],    msg=f'''Parameter {name} of model {model_class} seems not properly initialized''',    )





@require_vision
@require_torch
class 							lowerCAmelCase_		(	unittest.TestCase						):





    @cached_property
    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       return (
           AutoImageProcessor.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''     )
           if is_vision_available()
           else None
       )






    @slow
    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case   :							Dict						  =SwinForImageClassification.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''     ).to(_snake_case     )
       snake_case   :							Union[str, Any]						  =self.default_image_processor

       snake_case   :							Union[str, Any]						  =Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
       snake_case   :							int						  =image_processor(images=_snake_case,    return_tensors='''pt'''     ).to(_snake_case     )

       # forward pass
       with torch.no_grad():
          snake_case   :							Any						  =model(**_snake_case     )

       # verify the logits
       snake_case   :							Dict						  =torch.Size((1, 1_000)     )
       self.assertEqual(outputs.logits.shape,    _snake_case     )
       snake_case   :							int						  =torch.tensor([-0.0948, -0.6454, -0.0921]     ).to(_snake_case     )
       self.assertTrue(torch.allclose(outputs.logits[0, :3],    _snake_case,    atol=1E-4     )     )





@require_torch
class 							lowerCAmelCase_		(	unittest.TestCase  ,      a_						):
    __UpperCAmelCase							=	(SwinBackbone,) if is_torch_available() else ()
    __UpperCAmelCase							=	SwinConfig





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =SwinModelTester(self     )





