


import collections
import inspect
import unittest

from transformers import SwinConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel
 from transformers.models.swin.modeling_swin import SWIN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor




class A  :






      def __init__( self						:					Any      ,       lowercase_						:					int      ,       lowercase_						:					List[Any]=13      ,       lowercase_						:					Dict=32      ,       lowercase_						:					Union[str, Any]=2      ,       lowercase_						:					Tuple=3      ,       lowercase_						:					int=16      ,       lowercase_						:					Optional[Any]=[1, 2, 1]      ,       lowercase_						:					Optional[Any]=[2, 2, 4]      ,       lowercase_						:					List[str]=2      ,       lowercase_						:					Tuple=2.0      ,       lowercase_						:					Union[str, Any]=True      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					List[Any]=0.0      ,       lowercase_						:					int=0.1      ,       lowercase_						:					Union[str, Any]="gelu"      ,       lowercase_						:					Optional[int]=False      ,       lowercase_						:					Tuple=True      ,       lowercase_						:					Dict=0.02      ,       lowercase_						:					int=1E-5      ,       lowercase_						:					List[str]=True      ,       lowercase_						:					Tuple=None      ,       lowercase_						:					Optional[Any]=True      ,       lowercase_						:					Optional[int]=10      ,       lowercase_						:					Optional[int]=8      ,       lowercase_						:					List[Any]=["stage1", "stage2"]      ,       lowercase_						:					Any=[1, 2]      ,       ) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		Tuple          =parent
             _lowerCamelCase	:		int          =batch_size
             _lowerCamelCase	:		str          =image_size
             _lowerCamelCase	:		Optional[int]          =patch_size
             _lowerCamelCase	:		Optional[Any]          =num_channels
             _lowerCamelCase	:		List[str]          =embed_dim
             _lowerCamelCase	:		Dict          =depths
             _lowerCamelCase	:		Dict          =num_heads
             _lowerCamelCase	:		Any          =window_size
             _lowerCamelCase	:		Tuple          =mlp_ratio
             _lowerCamelCase	:		Tuple          =qkv_bias
             _lowerCamelCase	:		List[str]          =hidden_dropout_prob
             _lowerCamelCase	:		Any          =attention_probs_dropout_prob
             _lowerCamelCase	:		Union[str, Any]          =drop_path_rate
             _lowerCamelCase	:		Dict          =hidden_act
             _lowerCamelCase	:		Optional[int]          =use_absolute_embeddings
             _lowerCamelCase	:		List[str]          =patch_norm
             _lowerCamelCase	:		Union[str, Any]          =layer_norm_eps
             _lowerCamelCase	:		Tuple          =initializer_range
             _lowerCamelCase	:		List[Any]          =is_training
             _lowerCamelCase	:		Union[str, Any]          =scope
             _lowerCamelCase	:		Union[str, Any]          =use_labels
             _lowerCamelCase	:		int          =type_sequence_label_size
             _lowerCamelCase	:		List[Any]          =encoder_stride
             _lowerCamelCase	:		Dict          =out_features
             _lowerCamelCase	:		List[str]          =out_indices






      def 	lowerCamelCase	( self						:					Dict			) ->						int:
             """simple docstring"""




             _lowerCamelCase	:		Optional[int]          =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

             _lowerCamelCase	:		Any          =None
             if self.use_labels:
                    _lowerCamelCase	:		Dict          =ids_tensor([self.batch_size]      ,       self.type_sequence_label_size			)

             _lowerCamelCase	:		Dict          =self.get_config()

             return config, pixel_values, labels






      def 	lowerCamelCase	( self						:					List[Any]			) ->						Union[str, Any]:
             """simple docstring"""




             return SwinConfig(
                 image_size=self.image_size      ,       patch_size=self.patch_size      ,       num_channels=self.num_channels      ,       embed_dim=self.embed_dim      ,       depths=self.depths      ,       num_heads=self.num_heads      ,       window_size=self.window_size      ,       mlp_ratio=self.mlp_ratio      ,       qkv_bias=self.qkv_bias      ,       hidden_dropout_prob=self.hidden_dropout_prob      ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,       drop_path_rate=self.drop_path_rate      ,       hidden_act=self.hidden_act      ,       use_absolute_embeddings=self.use_absolute_embeddings      ,       path_norm=self.patch_norm      ,       layer_norm_eps=self.layer_norm_eps      ,       initializer_range=self.initializer_range      ,       encoder_stride=self.encoder_stride      ,       out_features=self.out_features      ,       out_indices=self.out_indices      ,       )






      def 	lowerCamelCase	( self						:					Dict      ,       lowercase_						:					str      ,       lowercase_						:					Tuple      ,       lowercase_						:					Any			) ->						Any:
             """simple docstring"""




             _lowerCamelCase	:		List[Any]          =SwinModel(config=snake_case_			)
             model.to(snake_case_			)
             model.eval()
             _lowerCamelCase	:		Tuple          =model(snake_case_			)

             _lowerCamelCase	:		int          =((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths			) - 1))
             _lowerCamelCase	:		List[str]          =int(config.embed_dim * 2 ** (len(config.depths			) - 1)			)

             self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, expected_seq_len, expected_dim)			)






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					Optional[int]      ,       lowercase_						:					Union[str, Any]      ,       lowercase_						:					int			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		Tuple          =SwinBackbone(config=snake_case_			)
             model.to(snake_case_			)
             model.eval()
             _lowerCamelCase	:		Optional[Any]          =model(snake_case_			)

             # verify hidden states
             self.parent.assertEqual(len(result.feature_maps			)      ,       len(config.out_features			)			)
             self.parent.assertListEqual(list(result.feature_maps[0].shape			)      ,       [self.batch_size, model.channels[0], 16, 16]			)

             # verify channels
             self.parent.assertEqual(len(model.channels			)      ,       len(config.out_features			)			)

             # verify backbone works with out_features=None
             _lowerCamelCase	:		Optional[int]          =None
             _lowerCamelCase	:		Union[str, Any]          =SwinBackbone(config=snake_case_			)
             model.to(snake_case_			)
             model.eval()
             _lowerCamelCase	:		Dict          =model(snake_case_			)

             # verify feature maps
             self.parent.assertEqual(len(result.feature_maps			)      ,       1			)
             self.parent.assertListEqual(list(result.feature_maps[0].shape			)      ,       [self.batch_size, model.channels[-1], 4, 4]			)

             # verify channels
             self.parent.assertEqual(len(model.channels			)      ,       1			)






      def 	lowerCamelCase	( self						:					Tuple      ,       lowercase_						:					List[Any]      ,       lowercase_						:					List[Any]      ,       lowercase_						:					str			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Tuple          =SwinForMaskedImageModeling(config=snake_case_			)
             model.to(snake_case_			)
             model.eval()
             _lowerCamelCase	:		List[str]          =model(snake_case_			)
             self.parent.assertEqual(
                 result.logits.shape      ,       (self.batch_size, self.num_channels, self.image_size, self.image_size)			)

             # test greyscale images
             _lowerCamelCase	:		Union[str, Any]          =1
             _lowerCamelCase	:		Optional[Any]          =SwinForMaskedImageModeling(snake_case_			)
             model.to(snake_case_			)
             model.eval()

             _lowerCamelCase	:		str          =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]			)
             _lowerCamelCase	:		Any          =model(snake_case_			)
             self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, 1, self.image_size, self.image_size)			)






      def 	lowerCamelCase	( self						:					Optional[int]      ,       lowercase_						:					str      ,       lowercase_						:					List[Any]      ,       lowercase_						:					Optional[int]			) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		List[str]          =self.type_sequence_label_size
             _lowerCamelCase	:		List[Any]          =SwinForImageClassification(snake_case_			)
             model.to(snake_case_			)
             model.eval()
             _lowerCamelCase	:		List[str]          =model(snake_case_      ,       labels=snake_case_			)
             self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.type_sequence_label_size)			)

             # test greyscale images
             _lowerCamelCase	:		Optional[Any]          =1
             _lowerCamelCase	:		Optional[int]          =SwinForImageClassification(snake_case_			)
             model.to(snake_case_			)
             model.eval()

             _lowerCamelCase	:		Dict          =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]			)
             _lowerCamelCase	:		Dict          =model(snake_case_			)
             self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.type_sequence_label_size)			)






      def 	lowerCamelCase	( self						:					Tuple			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[Any]          =self.prepare_config_and_inputs()
             (
                 _lowerCamelCase	
             )	:		List[str]          =config_and_inputs
             _lowerCamelCase	:		Optional[int]          ={'pixel_values': pixel_values}
             return config, inputs_dict




@require_torch
class A  (       UpperCAmelCase__   ,							UpperCAmelCase__   ,							unittest.TestCase					):
      UpperCamelCase__							:  Any				=(
          (
              SwinModel,
              SwinBackbone,
              SwinForImageClassification,
              SwinForMaskedImageModeling,
          )
          if is_torch_available()
          else ()
      )
      UpperCamelCase__							:  Optional[Any]				=(
          {"""feature-extraction""": SwinModel, """image-classification""": SwinForImageClassification}
          if is_torch_available()
          else {}
      )
      UpperCamelCase__							:  Optional[int]				=True

      UpperCamelCase__							:  List[Any]				=False
      UpperCamelCase__							:  Union[str, Any]				=False
      UpperCamelCase__							:  str				=False






      def 	lowerCamelCase	( self						:					Tuple			) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		List[Any]          =SwinModelTester(self			)
             _lowerCamelCase	:		Optional[Any]          =ConfigTester(self      ,       config_class=snake_case_      ,       embed_dim=37			)






      def 	lowerCamelCase	( self						:					str			) ->						int:
             """simple docstring"""




             self.create_and_test_config_common_properties()
             self.config_tester.create_and_test_config_to_json_string()
             self.config_tester.create_and_test_config_to_json_file()
             self.config_tester.create_and_test_config_from_and_save_pretrained()
             self.config_tester.create_and_test_config_with_num_labels()
             self.config_tester.check_config_can_be_init_without_params()
             self.config_tester.check_config_arguments_init()






      def 	lowerCamelCase	( self						:					str			) ->						List[str]:
             """simple docstring"""




             return






      def 	lowerCamelCase	( self						:					Union[str, Any]			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		Dict          =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*snake_case_			)






      @unittest.skip(reason='Got `CUDA error: misaligned address` with PyTorch 2.0.0.'			)
      def 	lowerCamelCase	( self						:					Tuple			) ->						List[str]:
             """simple docstring"""




             pass






      def 	lowerCamelCase	( self						:					List[str]			) ->						Union[str, Any]:
             """simple docstring"""




             super().test_training_gradient_checkpointing()






      def 	lowerCamelCase	( self						:					Optional[Any]			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		Union[str, Any]          =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_backbone(*snake_case_			)






      def 	lowerCamelCase	( self						:					Tuple			) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		List[str]          =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_image_modeling(*snake_case_			)






      def 	lowerCamelCase	( self						:					Any			) ->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		List[Any]          =self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_image_classification(*snake_case_			)






      @unittest.skip(reason='Swin does not use inputs_embeds'			)
      def 	lowerCamelCase	( self						:					int			) ->						Dict:
             """simple docstring"""




             pass






      @unittest.skip(reason='Swin Transformer does not use feedforward chunking'			)
      def 	lowerCamelCase	( self						:					Union[str, Any]			) ->						Tuple:
             """simple docstring"""




             pass






      def 	lowerCamelCase	( self						:					Tuple			) ->						Dict:
             """simple docstring"""




             _lowerCamelCase	:		Dict          =self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		Tuple          =model_class(snake_case_			)
                    self.assertIsInstance(model.get_input_embeddings()      ,       (nn.Module)			)
                    _lowerCamelCase	:		List[Any]          =model.get_output_embeddings()
                    self.assertTrue(x is None or isinstance(snake_case_      ,       nn.Linear			)			)






      def 	lowerCamelCase	( self						:					Dict			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[Any]          =self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		Any          =model_class(snake_case_			)
                    _lowerCamelCase	:		Union[str, Any]          =inspect.signature(model.forward			)
                    # signature.parameters is an OrderedDict => so arg_names order is deterministic
                    _lowerCamelCase	:		Optional[int]          =[*signature.parameters.keys()]

                    _lowerCamelCase	:		Optional[Any]          =['pixel_values']
                    self.assertListEqual(arg_names[:1]      ,       snake_case_			)






      def 	lowerCamelCase	( self						:					Tuple			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[int]          =self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase	:		str          =True

             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		Union[str, Any]          =True
                    _lowerCamelCase	:		List[str]          =False
                    _lowerCamelCase	:		List[str]          =True
                    _lowerCamelCase	:		Optional[Any]          =model_class(snake_case_			)
                    model.to(snake_case_			)
                    model.eval()
                    with torch.no_grad():
                           _lowerCamelCase	:		Tuple          =model(**self._prepare_for_class(snake_case_      ,       snake_case_			)			)
                    _lowerCamelCase	:		str          =outputs.attentions
                    _lowerCamelCase	:		Union[str, Any]          =len(self.model_tester.depths			)
                    self.assertEqual(len(snake_case_			)      ,       snake_case_			)

                    # check that output_attentions also work using config
                    del inputs_dict["output_attentions"]
                    _lowerCamelCase	:		Any          =True
                    _lowerCamelCase	:		Union[str, Any]          =config.window_size**2
                    _lowerCamelCase	:		List[str]          =model_class(snake_case_			)
                    model.to(snake_case_			)
                    model.eval()
                    with torch.no_grad():
                           _lowerCamelCase	:		Optional[Any]          =model(**self._prepare_for_class(snake_case_      ,       snake_case_			)			)
                    _lowerCamelCase	:		int          =outputs.attentions
                    self.assertEqual(len(snake_case_			)      ,       snake_case_			)

                    self.assertListEqual(
                        list(attentions[0].shape[-3:]			)      ,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]      ,       )
                    _lowerCamelCase	:		int          =len(snake_case_			)

                    # Check attention is always last and order is fine
                    _lowerCamelCase	:		Optional[Any]          =True
                    _lowerCamelCase	:		Optional[Any]          =True
                    _lowerCamelCase	:		Optional[Any]          =model_class(snake_case_			)
                    model.to(snake_case_			)
                    model.eval()
                    with torch.no_grad():
                           _lowerCamelCase	:		Optional[Any]          =model(**self._prepare_for_class(snake_case_      ,       snake_case_			)			)

                    # also another +1 for reshaped_hidden_states
                    _lowerCamelCase	:		List[str]          =1 if model_class.__name__ == 'SwinBackbone' else 2
                    self.assertEqual(out_len + added_hidden_states      ,       len(snake_case_			)			)

                    _lowerCamelCase	:		int          =outputs.attentions

                    self.assertEqual(len(snake_case_			)      ,       snake_case_			)

                    self.assertListEqual(
                        list(self_attentions[0].shape[-3:]			)      ,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]      ,       )






      def 	lowerCamelCase	( self						:					List[str]      ,       lowercase_						:					int      ,       lowercase_						:					Dict      ,       lowercase_						:					Any      ,       lowercase_						:					List[Any]			) ->						str:
             """simple docstring"""




             _lowerCamelCase	:		List[Any]          =model_class(snake_case_			)
             model.to(snake_case_			)
             model.eval()

             with torch.no_grad():
                    _lowerCamelCase	:		Union[str, Any]          =model(**self._prepare_for_class(snake_case_      ,       snake_case_			)			)

             _lowerCamelCase	:		str          =outputs.hidden_states

             _lowerCamelCase	:		int          =getattr(
                 self.model_tester      ,       'expected_num_hidden_layers'      ,       len(self.model_tester.depths			) + 1			)
             self.assertEqual(len(snake_case_			)      ,       snake_case_			)

             # Swin has a different seq_length
             _lowerCamelCase	:		Optional[Any]          =(
                 config.patch_size
                 if isinstance(config.patch_size      ,       collections.abc.Iterable			)
                 else (config.patch_size, config.patch_size)
             )

             _lowerCamelCase	:		Union[str, Any]          =(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

             self.assertListEqual(
                 list(hidden_states[0].shape[-2:]			)      ,       [num_patches, self.model_tester.embed_dim]      ,       )

             if not model_class.__name__ == "SwinBackbone":
                    _lowerCamelCase	:		Any          =outputs.reshaped_hidden_states
                    self.assertEqual(len(snake_case_			)      ,       snake_case_			)

                    _lowerCamelCase	:		Dict          =reshaped_hidden_states[0].shape
                    _lowerCamelCase	:		Optional[int]          =(
                        reshaped_hidden_states[0].view(snake_case_      ,       snake_case_      ,       height * width			).permute(0      ,       2      ,       1			)
                    )
                    self.assertListEqual(
                        list(reshaped_hidden_states.shape[-2:]			)      ,       [num_patches, self.model_tester.embed_dim]      ,       )






      def 	lowerCamelCase	( self						:					Union[str, Any]			) ->						Dict:
             """simple docstring"""




             _lowerCamelCase	:		List[Any]          =self.model_tester.prepare_config_and_inputs_for_common()

             _lowerCamelCase	:		Dict          =(
                 self.model_tester.image_size
                 if isinstance(self.model_tester.image_size      ,       collections.abc.Iterable			)
                 else (self.model_tester.image_size, self.model_tester.image_size)
             )

             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		int          =True
                    self.check_hidden_states_output(snake_case_      ,       snake_case_      ,       snake_case_      ,       snake_case_			)

                    # check that output_hidden_states also work using config
                    del inputs_dict["output_hidden_states"]
                    _lowerCamelCase	:		Union[str, Any]          =True

                    self.check_hidden_states_output(snake_case_      ,       snake_case_      ,       snake_case_      ,       snake_case_			)






      def 	lowerCamelCase	( self						:					Any			) ->						Any:
             """simple docstring"""




             _lowerCamelCase	:		int          =self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase	:		str          =3

             _lowerCamelCase	:		Tuple          =(
                 self.model_tester.image_size
                 if isinstance(self.model_tester.image_size      ,       collections.abc.Iterable			)
                 else (self.model_tester.image_size, self.model_tester.image_size)
             )
             _lowerCamelCase	:		str          =(
                 config.patch_size
                 if isinstance(config.patch_size      ,       collections.abc.Iterable			)
                 else (config.patch_size, config.patch_size)
             )

             _lowerCamelCase	:		Optional[Any]          =image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
             _lowerCamelCase	:		List[Any]          =image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		Union[str, Any]          =True
                    self.check_hidden_states_output(snake_case_      ,       snake_case_      ,       snake_case_      ,       (padded_height, padded_width)			)

                    # check that output_hidden_states also work using config
                    del inputs_dict["output_hidden_states"]
                    _lowerCamelCase	:		str          =True
                    self.check_hidden_states_output(snake_case_      ,       snake_case_      ,       snake_case_      ,       (padded_height, padded_width)			)






      @slow
      def 	lowerCamelCase	( self						:					Union[str, Any]			) ->						Optional[Any]:
             """simple docstring"""




             for model_name in SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    _lowerCamelCase	:		Any          =SwinModel.from_pretrained(snake_case_			)
                    self.assertIsNotNone(snake_case_			)






      def 	lowerCamelCase	( self						:					Optional[Any]			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		int          =self.model_tester.prepare_config_and_inputs_for_common()

             _lowerCamelCase	:		Optional[int]          =_config_zero_init(snake_case_			)
             for model_class in self.all_model_classes:
                    _lowerCamelCase	:		List[str]          =model_class(config=snake_case_			)
                    for name, param in model.named_parameters():
                           if "embeddings" not in name and param.requires_grad:
                                  self.assertIn(
                                      ((param.data.mean() * 1E9).round() / 1E9).item()      ,       [0.0, 1.0]      ,       msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''      ,       )




@require_vision
@require_torch
class A  (       unittest.TestCase					):






      @cached_property
      def 	lowerCamelCase	( self						:					Optional[Any]			) ->						List[str]:
             """simple docstring"""




             return (
                 AutoImageProcessor.from_pretrained('microsoft/swin-tiny-patch4-window7-224'			)
                 if is_vision_available()
                 else None
             )






      @slow
      def 	lowerCamelCase	( self						:					List[str]			) ->						Dict:
             """simple docstring"""




             _lowerCamelCase	:		List[str]          =SwinForImageClassification.from_pretrained('microsoft/swin-tiny-patch4-window7-224'			).to(snake_case_			)
             _lowerCamelCase	:		int          =self.default_image_processor

             _lowerCamelCase	:		Optional[int]          =Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'			)
             _lowerCamelCase	:		Tuple          =image_processor(images=snake_case_      ,       return_tensors='pt'			).to(snake_case_			)

             # forward pass
             with torch.no_grad():
                    _lowerCamelCase	:		List[Any]          =model(**snake_case_			)

             # verify the logits
             _lowerCamelCase	:		List[str]          =torch.Size((1, 1000)			)
             self.assertEqual(outputs.logits.shape      ,       snake_case_			)
             _lowerCamelCase	:		int          =torch.tensor([-0.0948, -0.6454, -0.0921]			).to(snake_case_			)
             self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,       snake_case_      ,       atol=1E-4			)			)




@require_torch
class A  (       unittest.TestCase   ,							UpperCAmelCase__					):
      UpperCamelCase__							:  Union[str, Any]				=(SwinBackbone,) if is_torch_available() else ()
      UpperCamelCase__							:  Union[str, Any]				=SwinConfig






      def 	lowerCamelCase	( self						:					Any			) ->						Tuple:
             """simple docstring"""




             _lowerCamelCase	:		Tuple          =SwinModelTester(self			)






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowerCamelCase						 =	logging.get_logger(__name__)

lowerCamelCase						 =	{
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}




class A  (       UpperCamelCase_					):
      UpperCamelCase__							:  int				='bigbird_pegasus'
      UpperCamelCase__							:  Tuple				=['past_key_values']
      UpperCamelCase__							:  str				={
          'num_attention_heads': 'encoder_attention_heads',
          'hidden_size': 'd_model',
          'attention_probs_dropout_prob': 'attention_dropout',
      }






      def __init__( self						:					Tuple      ,       lowercase_						:					int=9_6103      ,       lowercase_						:					Optional[Any]=4096      ,       lowercase_						:					Tuple=16      ,       lowercase_						:					int=4096      ,       lowercase_						:					Any=16      ,       lowercase_						:					str=16      ,       lowercase_						:					List[str]=4096      ,       lowercase_						:					Optional[Any]=16      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					List[Any]=0.0      ,       lowercase_						:					Optional[int]=True      ,       lowercase_						:					Tuple=True      ,       lowercase_						:					Dict="gelu_new"      ,       lowercase_						:					List[str]=1024      ,       lowercase_						:					Dict=0.1      ,       lowercase_						:					Union[str, Any]=0.0      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					str=0.02      ,       lowercase_						:					List[str]=2      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					int=True      ,       lowercase_						:					Union[str, Any]=0      ,       lowercase_						:					List[str]=2      ,       lowercase_						:					Optional[Any]=1      ,       lowercase_						:					Any="block_sparse"      ,       lowercase_						:					str=64      ,       lowercase_						:					Dict=3      ,       lowercase_						:					List[str]=False      ,       **lowercase_						:					List[str]      ,       ) ->						str:
             """simple docstring"""




             _lowerCamelCase	:		Optional[int]          =vocab_size
             _lowerCamelCase	:		Tuple          =max_position_embeddings
             _lowerCamelCase	:		Any          =d_model
             _lowerCamelCase	:		Optional[Any]          =encoder_ffn_dim
             _lowerCamelCase	:		Union[str, Any]          =encoder_layers
             _lowerCamelCase	:		int          =encoder_attention_heads
             _lowerCamelCase	:		Optional[Any]          =decoder_ffn_dim
             _lowerCamelCase	:		int          =decoder_layers
             _lowerCamelCase	:		str          =decoder_attention_heads
             _lowerCamelCase	:		Union[str, Any]          =dropout
             _lowerCamelCase	:		str          =attention_dropout
             _lowerCamelCase	:		int          =activation_dropout
             _lowerCamelCase	:		Union[str, Any]          =activation_function
             _lowerCamelCase	:		Optional[Any]          =init_std
             _lowerCamelCase	:		Optional[Any]          =encoder_layerdrop
             _lowerCamelCase	:		int          =decoder_layerdrop
             _lowerCamelCase	:		Optional[int]          =classifier_dropout
             _lowerCamelCase	:		int          =use_cache
             _lowerCamelCase	:		Dict          =encoder_layers
             _lowerCamelCase	:		List[str]          =scale_embedding  # scale factor will be sqrt(d_model) if True

             # extra config
             _lowerCamelCase	:		Tuple          =attention_type
             _lowerCamelCase	:		str          =block_size
             _lowerCamelCase	:		Optional[int]          =num_random_blocks
             _lowerCamelCase	:		Optional[int]          =use_bias

             super().__init__(
                 pad_token_id=lowercase_      ,       bos_token_id=lowercase_      ,       eos_token_id=lowercase_      ,       is_encoder_decoder=lowercase_      ,       decoder_start_token_id=lowercase_      ,       **lowercase_      ,       )




class A  (       UpperCamelCase_					):






      @property
      def 	lowerCamelCase	( self						:					Dict			) ->						Mapping[str, Mapping[int, str]]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Dict          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                        ]			)

                    if self.use_past:
                           _lowerCamelCase	:		str          ={0: 'batch'}
                           _lowerCamelCase	:		Optional[int]          ={0: 'batch', 1: 'past_decoder_sequence + sequence'}
                    else:
                           _lowerCamelCase	:		Optional[Any]          ={0: 'batch', 1: 'decoder_sequence'}
                           _lowerCamelCase	:		List[Any]          ={0: 'batch', 1: 'decoder_sequence'}

                    if self.use_past:
                           self.fill_with_past_key_values_(lowercase_      ,       direction='inputs'			)
             elif self.task == "causal-lm":
                    # TODO: figure this case out.
                    _lowerCamelCase	:		Union[str, Any]          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                        ]			)
                    if self.use_past:
                           _lowerCamelCase     ,					_lowerCamelCase	:		Union[str, Any]          =self.num_layers
                           for i in range(lowercase_			):
                                  _lowerCamelCase	:		Dict          ={0: 'batch', 2: 'past_sequence + sequence'}
                                  _lowerCamelCase	:		List[str]          ={0: 'batch', 2: 'past_sequence + sequence'}
             else:
                    _lowerCamelCase	:		Optional[int]          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                            ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
                            ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
                        ]			)

             return common_inputs






      @property
      def 	lowerCamelCase	( self						:					Dict			) ->						Mapping[str, Mapping[int, str]]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Dict          =super().outputs
             else:
                    _lowerCamelCase	:		Optional[Any]          =super(lowercase_      ,       self			).outputs
                    if self.use_past:
                           _lowerCamelCase     ,					_lowerCamelCase	:		str          =self.num_layers
                           for i in range(lowercase_			):
                                  _lowerCamelCase	:		Tuple          ={0: 'batch', 2: 'past_sequence + sequence'}
                                  _lowerCamelCase	:		List[Any]          ={0: 'batch', 2: 'past_sequence + sequence'}
             return common_outputs






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		int          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)

             # Generate decoder inputs
             _lowerCamelCase	:		str          =seq_length if not self.use_past else 1
             _lowerCamelCase	:		List[Any]          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)
             _lowerCamelCase	:		List[Any]          ={F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
             _lowerCamelCase	:		Union[str, Any]          =dict(**lowercase_      ,       **lowercase_			)

             if self.use_past:
                    if not is_torch_available():
                           raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'			)
                    else:
                           import torch
                    _lowerCamelCase     ,					_lowerCamelCase	:		int          =common_inputs['input_ids'].shape
                    _lowerCamelCase	:		List[Any]          =common_inputs['decoder_input_ids'].shape[1]
                    _lowerCamelCase     ,					_lowerCamelCase	:		Union[str, Any]          =self.num_attention_heads
                    _lowerCamelCase	:		Dict          =(
                        batch,
                        num_encoder_attention_heads,
                        encoder_seq_length,
                        self._config.hidden_size // num_encoder_attention_heads,
                    )
                    _lowerCamelCase	:		List[Any]          =decoder_seq_length + 3
                    _lowerCamelCase	:		int          =(
                        batch,
                        num_decoder_attention_heads,
                        decoder_past_length,
                        self._config.hidden_size // num_decoder_attention_heads,
                    )

                    _lowerCamelCase	:		Union[str, Any]          =torch.cat(
                        [common_inputs['decoder_attention_mask'], torch.ones(lowercase_      ,       lowercase_			)]      ,       dim=1			)

                    _lowerCamelCase	:		Optional[Any]          =[]
                    # If the number of encoder and decoder layers are present in the model configuration, both are considered
                    _lowerCamelCase     ,					_lowerCamelCase	:		Tuple          =self.num_layers
                    _lowerCamelCase	:		Optional[int]          =min(lowercase_      ,       lowercase_			)
                    _lowerCamelCase	:		int          =max(lowercase_      ,       lowercase_			) - min_num_layers
                    _lowerCamelCase	:		str          ='encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

                    for _ in range(lowercase_			):
                           common_inputs["past_key_values"].append(
                               (
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                               )			)
                    # TODO: test this.
                    _lowerCamelCase	:		List[Any]          =encoder_shape if remaining_side_name == 'encoder' else decoder_shape
                    for _ in range(lowercase_      ,       lowercase_			):
                           common_inputs["past_key_values"].append((torch.zeros(lowercase_			), torch.zeros(lowercase_			))			)
             return common_inputs






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		str          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)

             if self.use_past:
                    if not is_torch_available():
                           raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'			)
                    else:
                           import torch
                    _lowerCamelCase     ,					_lowerCamelCase	:		Optional[int]          =common_inputs['input_ids'].shape
                    # Not using the same length for past_key_values
                    _lowerCamelCase	:		Union[str, Any]          =seqlen + 2
                    _lowerCamelCase     ,					_lowerCamelCase	:		List[Any]          =self.num_layers
                    _lowerCamelCase     ,					_lowerCamelCase	:		Any          =self.num_attention_heads
                    _lowerCamelCase	:		List[str]          =(
                        batch,
                        num_encoder_attention_heads,
                        past_key_values_length,
                        self._config.hidden_size // num_encoder_attention_heads,
                    )

                    _lowerCamelCase	:		Tuple          =common_inputs['attention_mask'].dtype
                    _lowerCamelCase	:		Tuple          =torch.cat(
                        [common_inputs['attention_mask'], torch.ones(lowercase_      ,       lowercase_      ,       dtype=lowercase_			)]      ,       dim=1			)
                    _lowerCamelCase	:		Dict          =[
                        (torch.zeros(lowercase_			), torch.zeros(lowercase_			)) for _ in range(lowercase_			)
                    ]
             return common_inputs






      def 	lowerCamelCase	( self						:					List[str]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[Any]          =compute_effective_axis_dimension(
                 lowercase_      ,       fixed_dimension=OnnxConfig.default_fixed_batch      ,       num_token_to_add=0			)

             # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
             _lowerCamelCase	:		Tuple          =tokenizer.num_special_tokens_to_add(lowercase_			)
             _lowerCamelCase	:		Any          =compute_effective_axis_dimension(
                 lowercase_      ,       fixed_dimension=OnnxConfig.default_fixed_sequence      ,       num_token_to_add=lowercase_			)

             # Generate dummy inputs according to compute batch and sequence
             _lowerCamelCase	:		Any          =[' '.join([tokenizer.unk_token]			) * seq_length] * batch_size
             _lowerCamelCase	:		int          =dict(tokenizer(lowercase_      ,       return_tensors=lowercase_			)			)
             return common_inputs






      def 	lowerCamelCase	( self						:					Any      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Union[str, Any]          =self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)

             elif self.task == "causal-lm":
                    _lowerCamelCase	:		List[str]          =self._generate_dummy_inputs_for_causal_lm(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)
             else:
                    _lowerCamelCase	:		List[Any]          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)

             return common_inputs






      def 	lowerCamelCase	( self						:					List[Any]      ,       lowercase_						:					Union[str, Any]      ,       lowercase_						:					Optional[int]      ,       lowercase_						:					Dict      ,       lowercase_						:					Tuple			) ->						Dict:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Union[str, Any]          =super()._flatten_past_key_values_(lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)
             else:
                    _lowerCamelCase	:		List[str]          =super(lowercase_      ,       self			)._flatten_past_key_values_(
                        lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)


