




"""simple docstring"""




import inspect
import unittest

from transformers import EfficientNetConfig
from transformers.testing_utils import is_pipeline_test, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import EfficientNetForImageClassification, EfficientNetModel
		from transformers.models.efficientnet.modeling_efficientnet import EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from PIL import Image

		from transformers import AutoImageProcessor
class 		UpperCAmelCase       :







						'''simple docstring'''





						def __init__(						self							:					Optional[int]	,     __lowercase							:					Optional[int]	,     __lowercase							:					List[str]=13	,     __lowercase							:					List[str]=32	,     __lowercase							:					Optional[int]=3	,     __lowercase							:					Optional[int]=[3, 3, 5]	,     __lowercase							:					Optional[int]=[32, 16, 24]	,     __lowercase							:					str=[16, 24, 40]	,     __lowercase							:					List[Any]=[1, 1, 2]	,     __lowercase							:					List[str]=[1, 1, 2]	,     __lowercase							:					int=[1, 6, 6]	,     __lowercase							:					List[Any]=True	,     __lowercase							:					Any=True	,     __lowercase							:					List[Any]=37	,     __lowercase							:					Optional[Any]="gelu"	,     __lowercase							:					Optional[int]=10	,     ):






							"""simple docstring"""






							snake_case_		   =					parent
							snake_case_		   =					batch_size
							snake_case_		   =					image_size
							snake_case_		   =					num_channels
							snake_case_		   =					kernel_sizes
							snake_case_		   =					in_channels
							snake_case_		   =					out_channels
							snake_case_		   =					strides
							snake_case_		   =					num_block_repeats
							snake_case_		   =					expand_ratios
							snake_case_		   =					is_training
							snake_case_		   =					hidden_act
							snake_case_		   =					num_labels
							snake_case_		   =					use_labels






						def 	snake_case__						(						self							:					Any						):






							"""simple docstring"""






							snake_case_		   =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

							snake_case_		   =					None
							if self.use_labels:
								snake_case_		   =					ids_tensor([self.batch_size]	,     self.num_labels						)

							snake_case_		   =					self.get_config()
							return config, pixel_values, labels






						def 	snake_case__						(						self							:					Optional[Any]						):






							"""simple docstring"""






							return EfficientNetConfig(
							    num_channels=self.num_channels	,     kernel_sizes=self.kernel_sizes	,     in_channels=self.in_channels	,     out_channels=self.out_channels	,     strides=self.strides	,     num_block_repeats=self.num_block_repeats	,     expand_ratios=self.expand_ratios	,     hidden_act=self.hidden_act	,     num_labels=self.num_labels	,     )






						def 	snake_case__						(						self							:					Dict	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					List[str]	,     __lowercase							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					EfficientNetModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase						)
							# expected last hidden states: B, C, H // 4, W // 4
							self.parent.assertEqual(
							    result.last_hidden_state.shape	,     (self.batch_size, config.hidden_dim, self.image_size // 4, self.image_size // 4)	,     )






						def 	snake_case__						(						self							:					Optional[int]	,     __lowercase							:					Dict	,     __lowercase							:					Optional[Any]	,     __lowercase							:					str						):






							"""simple docstring"""






							snake_case_		   =					EfficientNetForImageClassification(__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.num_labels)						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					self.prepare_config_and_inputs()
							snake_case_   ,				snake_case_   ,				snake_case_		   =					config_and_inputs
							snake_case_		   =					{"pixel_values": pixel_values}
							return config, inputs_dict







@require_torch
class 		UpperCAmelCase       (    UpperCAmelCase__	,       UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       (EfficientNetModel, EfficientNetForImageClassification) if is_torch_available() else ()
						lowerCAmelCase_							=       (
						    {'''feature-extraction''': EfficientNetModel, '''image-classification''': EfficientNetForImageClassification}
						    if is_torch_available()
						    else {}
						)

						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					EfficientNetModelTester(self						)
							snake_case_		   =					ConfigTester(
							    self	,     config_class=__lowercase	,     has_text_modality=__lowercase	,     hidden_size=37						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							self.create_and_test_config_common_properties()
							self.config_tester.create_and_test_config_to_json_string()
							self.config_tester.create_and_test_config_to_json_file()
							self.config_tester.create_and_test_config_from_and_save_pretrained()
							self.config_tester.create_and_test_config_with_num_labels()
							self.config_tester.check_config_can_be_init_without_params()
							self.config_tester.check_config_arguments_init()






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							return






						@unittest.skip(reason="EfficientNet does not use inputs_embeds"						)
						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							pass






						@unittest.skip(reason="EfficientNet does not support input and output embeddings"						)
						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							pass






						@unittest.skip(reason="EfficientNet does not use feedforward chunking"						)
						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							pass






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_   ,				snake_case_		   =					self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								snake_case_		   =					model_class(__lowercase						)
								snake_case_		   =					inspect.signature(model.forward						)
								# signature.parameters is an OrderedDict => so arg_names order is deterministic
								snake_case_		   =					[*signature.parameters.keys()]

								snake_case_		   =					["pixel_values"]
								self.assertListEqual(arg_names[:1]	,     __lowercase						)






						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							def check_hidden_states_output(__lowercase							:					Tuple	,     __lowercase							:					Any	,     __lowercase							:					Optional[int]						):
								snake_case_		   =					model_class(__lowercase						)
								model.to(__lowercase						)
								model.eval()

								with torch.no_grad():
									snake_case_		   =					model(**self._prepare_for_class(__lowercase	,     __lowercase						)						)

								snake_case_		   =					outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states
								snake_case_		   =					sum(config.num_block_repeats						) * 4
								self.assertEqual(len(__lowercase						)	,     __lowercase						)

								# EfficientNet's feature maps are of shape (batch_size, num_channels, height, width)
								self.assertListEqual(
								    list(hidden_states[0].shape[-2:]						)	,     [self.model_tester.image_size // 2, self.model_tester.image_size // 2]	,     )

							snake_case_   ,				snake_case_		   =					self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								snake_case_		   =					True
								check_hidden_states_output(__lowercase	,     __lowercase	,     __lowercase						)

								# check that output_hidden_states also work using config
								del inputs_dict["output_hidden_states"]
								snake_case_		   =					True

								check_hidden_states_output(__lowercase	,     __lowercase	,     __lowercase						)






						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*__lowercase						)






						@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."						)
						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							pass






						@slow
						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							for model_name in EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		   =					EfficientNetModel.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)






						@is_pipeline_test
						@require_vision
						@slow
						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							super().test_pipeline_image_classification()







def 						lowerCamelCase__   (		):







	'''simple docstring'''





	snake_case_		   =					Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"							)
	return image






@require_torch
@require_vision
class 		UpperCAmelCase       (    unittest.TestCase      ):







						'''simple docstring'''





						@cached_property
						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							return AutoImageProcessor.from_pretrained("google/efficientnet-b7"						) if is_vision_available() else None






						@slow
						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							snake_case_		   =					EfficientNetForImageClassification.from_pretrained("google/efficientnet-b7"						).to(__lowercase						)

							snake_case_		   =					self.default_image_processor
							snake_case_		   =					prepare_img()
							snake_case_		   =					image_processor(images=__lowercase	,     return_tensors="pt"						).to(__lowercase						)

							# forward pass
							with torch.no_grad():
								snake_case_		   =					model(**__lowercase						)

							# verify the logits
							snake_case_		   =					torch.Size((1, 10_00)						)
							self.assertEqual(outputs.logits.shape	,     __lowercase						)

							snake_case_		   =					torch.tensor([0.0001, 0.0002, 0.0002]						).to(__lowercase						)
							self.assertTrue(torch.allclose(outputs.logits[0, :3]	,     __lowercase	,     atol=1E-4						)						)






import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import RobertaConfig, RobertaForMaskedLM, RobertaForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("0.9.0"):
		raise Exception("requires fairseq >= 0.9.0")


logging.set_verbosity_info()
lowercase__    :					Dict					=						logging.get_logger(__name__)

lowercase__    :					Any					=						"Hello world! cécé herlolip"


def 						lowerCamelCase__   (		_A					,     _A					,     _A							):







	'''simple docstring'''





	snake_case_		   =					FairseqRobertaModel.from_pretrained(_A							)
	roberta.eval()  # disable dropout
	snake_case_		   =					roberta.model.encoder.sentence_encoder
	snake_case_		   =					RobertaConfig(
	    vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings					,     hidden_size=roberta.args.encoder_embed_dim					,     num_hidden_layers=roberta.args.encoder_layers					,     num_attention_heads=roberta.args.encoder_attention_heads					,     intermediate_size=roberta.args.encoder_ffn_embed_dim					,     max_position_embeddings=514					,     type_vocab_size=1					,     layer_norm_eps=1E-5					,     )
	if classification_head:
		snake_case_		   =					roberta.model.classification_heads["mnli"].out_proj.weight.shape[0]
	print("Our BERT config:"					,     _A							)

	snake_case_		   =					RobertaForSequenceClassification(_A							) if classification_head else RobertaForMaskedLM(_A							)
	model.eval()

	# Now let's copy all the weights.
	# Embeddings
	snake_case_		   =					roberta_sent_encoder.embed_tokens.weight
	snake_case_		   =					roberta_sent_encoder.embed_positions.weight
	snake_case_		   =					torch.zeros_like(
	    model.roberta.embeddings.token_type_embeddings.weight							)  # just zero them out b/c RoBERTa doesn't use them.
	snake_case_		   =					roberta_sent_encoder.emb_layer_norm.weight
	snake_case_		   =					roberta_sent_encoder.emb_layer_norm.bias

	for i in range(config.num_hidden_layers							):
		# Encoder: start of layer
		snake_case_		   =					model.roberta.encoder.layer[i]
		snake_case_		   =					roberta_sent_encoder.layers[i]

		# self attention
		snake_case_		   =					layer.attention.self
		assert (
		    roberta_layer.self_attn.k_proj.weight.data.shape
		    == roberta_layer.self_attn.q_proj.weight.data.shape
		    == roberta_layer.self_attn.v_proj.weight.data.shape
		    == torch.Size((config.hidden_size, config.hidden_size)							)
		)

		snake_case_		   =					roberta_layer.self_attn.q_proj.weight
		snake_case_		   =					roberta_layer.self_attn.q_proj.bias
		snake_case_		   =					roberta_layer.self_attn.k_proj.weight
		snake_case_		   =					roberta_layer.self_attn.k_proj.bias
		snake_case_		   =					roberta_layer.self_attn.v_proj.weight
		snake_case_		   =					roberta_layer.self_attn.v_proj.bias

		# self-attention output
		snake_case_		   =					layer.attention.output
		assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
		snake_case_		   =					roberta_layer.self_attn.out_proj.weight
		snake_case_		   =					roberta_layer.self_attn.out_proj.bias
		snake_case_		   =					roberta_layer.self_attn_layer_norm.weight
		snake_case_		   =					roberta_layer.self_attn_layer_norm.bias

		# intermediate
		snake_case_		   =					layer.intermediate
		assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
		snake_case_		   =					roberta_layer.fca.weight
		snake_case_		   =					roberta_layer.fca.bias

		# output
		snake_case_		   =					layer.output
		assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
		snake_case_		   =					roberta_layer.fca.weight
		snake_case_		   =					roberta_layer.fca.bias
		snake_case_		   =					roberta_layer.final_layer_norm.weight
		snake_case_		   =					roberta_layer.final_layer_norm.bias
		# end of layer

	if classification_head:
		snake_case_		   =					roberta.model.classification_heads["mnli"].dense.weight
		snake_case_		   =					roberta.model.classification_heads["mnli"].dense.bias
		snake_case_		   =					roberta.model.classification_heads["mnli"].out_proj.weight
		snake_case_		   =					roberta.model.classification_heads["mnli"].out_proj.bias
	else:
		# LM Head
		snake_case_		   =					roberta.model.encoder.lm_head.dense.weight
		snake_case_		   =					roberta.model.encoder.lm_head.dense.bias
		snake_case_		   =					roberta.model.encoder.lm_head.layer_norm.weight
		snake_case_		   =					roberta.model.encoder.lm_head.layer_norm.bias
		snake_case_		   =					roberta.model.encoder.lm_head.weight
		snake_case_		   =					roberta.model.encoder.lm_head.bias

	# Let's check that we get the same results.
	snake_case_		   =					roberta.encode(_A							).unsqueeze(0							)  # batch of size 1

	snake_case_		   =					model(_A							)[0]
	if classification_head:
		snake_case_		   =					roberta.model.classification_heads["mnli"](roberta.extract_features(_A							)							)
	else:
		snake_case_		   =					roberta.model(_A							)[0]
	print(our_output.shape					,     their_output.shape							)
	snake_case_		   =					torch.max(torch.abs(our_output - their_output							)							).item()
	print(f"max_absolute_diff = {max_absolute_diff}"							)  # ~ 1e-7
	snake_case_		   =					torch.allclose(_A					,     _A					,     atol=1E-3							)
	print("Do both models output the same tensors?"					,     "🔥" if success else "💩"							)
	if not success:
		raise Exception("Something went wRoNg"							)

	pathlib.Path(_A							).mkdir(parents=_A					,     exist_ok=_A							)
	print(f"Saving model to {pytorch_dump_folder_path}"							)
	model.save_pretrained(_A							)


if __name__ == "__main__":
		lowercase__    :					int					=						argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
		)
		parser.add_argument(
		    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
		)
		parser.add_argument(
		    "--classification_head", action="store_true", help="Whether to convert a final classification head."
		)
		lowercase__    :					Optional[Any]					=						parser.parse_args()
		convert_roberta_checkpoint_to_pytorch(
		    args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
		)
