




import argparse
import os

import align
import numpy as np
import requests
import tensorflow as tf
import torch
from PIL import Image
from tokenizer import Tokenizer

from transformers import (
    AlignConfig,
    AlignModel,
    AlignProcessor,
    BertConfig,
    BertTokenizer,
    EfficientNetConfig,
    EfficientNetImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a								=							logging.get_logger(__name__)





def 							lowercase   (snake_case__				:	Union[str, Any]	)       ->					Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			tf.image.resize(snake_case__ ,    (346, 346)	)
				lowerCAmelCase		=			tf.image.crop_to_bounding_box(snake_case__ ,    (346 - 289) // 2 ,    (346 - 289) // 2 ,    289 ,    289	)
				return image





def 							lowercase   ()       ->					Any:


				'''simple docstring'''






				lowerCAmelCase		=			EfficientNetConfig.from_pretrained("""google/efficientnet-b7"""	)
				lowerCAmelCase		=			289
				lowerCAmelCase		=			640
				lowerCAmelCase		=			{"""0""": """LABEL_0""", """1""": """LABEL_1"""}
				lowerCAmelCase		=			{"""LABEL_0""": 0, """LABEL_1""": 1}
				lowerCAmelCase		=			[]

				lowerCAmelCase		=			BertConfig()
				lowerCAmelCase		=			AlignConfig.from_text_vision_configs(
				    text_config=snake_case__ ,    vision_config=snake_case__ ,    projection_dim=640	)
				return config





def 							lowercase   ()       ->					List[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			"""http://images.cocodataset.org/val2017/000000039769.jpg"""
				lowerCAmelCase		=			Image.open(requests.get(snake_case__ ,    stream=snake_case__	).raw	)
				return im





def 							lowercase   ()       ->					Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			EfficientNetImageProcessor(
				    do_center_crop=snake_case__ ,    rescale_factor=1 / 127.5 ,    rescale_offset=snake_case__ ,    do_normalize=snake_case__ ,    include_top=snake_case__ ,    resample=Image.BILINEAR ,    )
				lowerCAmelCase		=			BertTokenizer.from_pretrained("""bert-base-uncased"""	)
				lowerCAmelCase		=			64
				lowerCAmelCase		=			AlignProcessor(image_processor=snake_case__ ,    tokenizer=snake_case__	)
				return processor





def 							lowercase   (snake_case__				:	int	)       ->					str:


				'''simple docstring'''






				lowerCAmelCase		=			[v.split("""_"""	)[0].split("""block"""	)[1] for v in original_param_names if v.startswith("""block"""	)]
				lowerCAmelCase		=			list(set(snake_case__	)	)
				lowerCAmelCase		=			sorted(snake_case__	)
				lowerCAmelCase		=			len(snake_case__	)
				lowerCAmelCase		=			{b: str(snake_case__	) for b, i in zip(snake_case__ ,    range(snake_case__	)	)}

				lowerCAmelCase		=			[]
				rename_keys.append(("""stem_conv/kernel:0""", """embeddings.convolution.weight""")	)
				rename_keys.append(("""stem_bn/gamma:0""", """embeddings.batchnorm.weight""")	)
				rename_keys.append(("""stem_bn/beta:0""", """embeddings.batchnorm.bias""")	)
				rename_keys.append(("""stem_bn/moving_mean:0""", """embeddings.batchnorm.running_mean""")	)
				rename_keys.append(("""stem_bn/moving_variance:0""", """embeddings.batchnorm.running_var""")	)

				for b in block_names:
								lowerCAmelCase		=			block_name_mapping[b]
								rename_keys.append((f'''block{b}_expand_conv/kernel:0''', f'''encoder.blocks.{hf_b}.expansion.expand_conv.weight''')	)
								rename_keys.append((f'''block{b}_expand_bn/gamma:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.weight''')	)
								rename_keys.append((f'''block{b}_expand_bn/beta:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.bias''')	)
								rename_keys.append(
								    (f'''block{b}_expand_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_expand_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.running_var''')	)
								rename_keys.append(
								    (f'''block{b}_dwconv/depthwise_kernel:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_conv.weight''')	)
								rename_keys.append((f'''block{b}_bn/gamma:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.weight''')	)
								rename_keys.append((f'''block{b}_bn/beta:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.bias''')	)
								rename_keys.append(
								    (f'''block{b}_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_var''')	)

								rename_keys.append((f'''block{b}_se_reduce/kernel:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.reduce.weight''')	)
								rename_keys.append((f'''block{b}_se_reduce/bias:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.reduce.bias''')	)
								rename_keys.append((f'''block{b}_se_expand/kernel:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.expand.weight''')	)
								rename_keys.append((f'''block{b}_se_expand/bias:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.expand.bias''')	)
								rename_keys.append(
								    (f'''block{b}_project_conv/kernel:0''', f'''encoder.blocks.{hf_b}.projection.project_conv.weight''')	)
								rename_keys.append((f'''block{b}_project_bn/gamma:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.weight''')	)
								rename_keys.append((f'''block{b}_project_bn/beta:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.bias''')	)
								rename_keys.append(
								    (f'''block{b}_project_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_project_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.running_var''')	)

				lowerCAmelCase		=			{}
				for item in rename_keys:
								if item[0] in original_param_names:
												lowerCAmelCase		=			"""vision_model.""" + item[1]

    # BERT text encoder
				lowerCAmelCase		=			[]
				lowerCAmelCase		=			"""tf_bert_model/bert"""
				lowerCAmelCase		=			"""text_model"""
				for i in range(12	):
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/query/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.query.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/query/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.query.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/key/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.key.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/key/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.key.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/value/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.value.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/value/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.value.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/dense/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.dense.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/dense/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.dense.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/LayerNorm/gamma:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.LayerNorm.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/LayerNorm/beta:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.LayerNorm.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/intermediate/dense/kernel:0''',
								        f'''{new}.encoder.layer.{i}.intermediate.dense.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/intermediate/dense/bias:0''',
								        f'''{new}.encoder.layer.{i}.intermediate.dense.bias''',
								    )	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/dense/kernel:0''', f'''{new}.encoder.layer.{i}.output.dense.weight''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/dense/bias:0''', f'''{new}.encoder.layer.{i}.output.dense.bias''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/LayerNorm/gamma:0''', f'''{new}.encoder.layer.{i}.output.LayerNorm.weight''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/LayerNorm/beta:0''', f'''{new}.encoder.layer.{i}.output.LayerNorm.bias''')	)

				rename_keys.append((f'''{old}/embeddings/word_embeddings/weight:0''', f'''{new}.embeddings.word_embeddings.weight''')	)
				rename_keys.append(
				    (f'''{old}/embeddings/position_embeddings/embeddings:0''', f'''{new}.embeddings.position_embeddings.weight''')	)
				rename_keys.append(
				    (f'''{old}/embeddings/token_type_embeddings/embeddings:0''', f'''{new}.embeddings.token_type_embeddings.weight''')	)
				rename_keys.append((f'''{old}/embeddings/LayerNorm/gamma:0''', f'''{new}.embeddings.LayerNorm.weight''')	)
				rename_keys.append((f'''{old}/embeddings/LayerNorm/beta:0''', f'''{new}.embeddings.LayerNorm.bias''')	)

				rename_keys.append((f'''{old}/pooler/dense/kernel:0''', f'''{new}.pooler.dense.weight''')	)
				rename_keys.append((f'''{old}/pooler/dense/bias:0''', f'''{new}.pooler.dense.bias''')	)
				rename_keys.append(("""dense/kernel:0""", """text_projection.weight""")	)
				rename_keys.append(("""dense/bias:0""", """text_projection.bias""")	)
				rename_keys.append(("""dense/bias:0""", """text_projection.bias""")	)
				rename_keys.append(("""temperature:0""", """temperature""")	)

				for item in rename_keys:
								if item[0] in original_param_names:
												lowerCAmelCase		=			item[1]
				return key_mapping





def 							lowercase   (snake_case__				:	int ,    snake_case__				:	List[Any] ,    snake_case__				:	Any	)       ->					List[Any]:


				'''simple docstring'''






				list(hf_params.keys()	)

				for key, value in tf_params.items():
								if key not in key_mapping:
												continue

								lowerCAmelCase		=			key_mapping[key]
								if "_conv" in key and "kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	).permute(3 ,    2 ,    0 ,    1	)
								elif "embeddings" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	)
								elif "depthwise_kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	).permute(2 ,    3 ,    0 ,    1	)
								elif "kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(np.transpose(snake_case__	)	)
								elif "temperature" in key:
												lowerCAmelCase		=			value
								elif "bn/gamma" or "bn/beta" in key:
												lowerCAmelCase		=			torch.from_numpy(np.transpose(snake_case__	)	).squeeze()
								else:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	)

								# Replace HF parameters with original TF model parameters
								hf_params[hf_key].copy_(snake_case__	)





@torch.no_grad()
def 							lowercase   (snake_case__				:	Dict ,    snake_case__				:	Dict ,    snake_case__				:	str ,    snake_case__				:	Union[str, Any]	)       ->					List[str]:


				'''simple docstring'''






				lowerCAmelCase		=			64
				lowerCAmelCase		=			Tokenizer(snake_case__	)
				lowerCAmelCase		=			align.Align("""efficientnet-b7""" ,    """bert-base""" ,    640 ,    snake_case__ ,    tok.get_vocab_size()	)
				original_model.compile()
				original_model.load_weights(snake_case__	)

				lowerCAmelCase		=			original_model.trainable_variables
				lowerCAmelCase		=			original_model.non_trainable_variables
				lowerCAmelCase		=			{param.name: param.numpy() for param in tf_params}
				for param in tf_non_train_params:
								lowerCAmelCase		=			param.numpy()
				lowerCAmelCase		=			list(tf_params.keys()	)

				# Load HuggingFace model
				lowerCAmelCase		=			get_align_config()
				lowerCAmelCase		=			AlignModel(snake_case__	).eval()
				lowerCAmelCase		=			hf_model.state_dict()

				# Create src-to-dst parameter name mapping dictionary
				print("""Converting parameters..."""	)
				lowerCAmelCase		=			rename_keys(snake_case__	)
				replace_params(snake_case__ ,    snake_case__ ,    snake_case__	)

				# Initialize processor
				lowerCAmelCase		=			get_processor()
				lowerCAmelCase		=			processor(
				    images=prepare_img() ,    text="""A picture of a cat""" ,    padding="""max_length""" ,    max_length=64 ,    return_tensors="""pt"""	)

				# HF model inference
				hf_model.eval()
				with torch.no_grad():
								lowerCAmelCase		=			hf_model(**snake_case__	)

				lowerCAmelCase		=			outputs.image_embeds.detach().numpy()
				lowerCAmelCase		=			outputs.text_embeds.detach().numpy()

				# Original model inference
				lowerCAmelCase		=			False
				lowerCAmelCase		=			EfficientNetImageProcessor(
				    do_center_crop=snake_case__ ,    do_rescale=snake_case__ ,    do_normalize=snake_case__ ,    include_top=snake_case__ ,    resample=Image.BILINEAR ,    )
				lowerCAmelCase		=			tf_image_processor(images=prepare_img() ,    return_tensors="""tf""" ,    data_format="""channels_last"""	)["""pixel_values"""]
				lowerCAmelCase		=			tok(tf.constant(["""A picture of a cat"""]	)	)

				lowerCAmelCase		=			original_model.image_encoder(snake_case__ ,    training=snake_case__	)
				lowerCAmelCase		=			original_model.text_encoder(snake_case__ ,    training=snake_case__	)

				lowerCAmelCase		=			tf.nn.la_normalize(snake_case__ ,    axis=-1	)
				lowerCAmelCase		=			tf.nn.la_normalize(snake_case__ ,    axis=-1	)

				# Check whether original and HF model outputs match  -> np.allclose
				if not np.allclose(snake_case__ ,    snake_case__ ,    atol=1e-3	):
								raise ValueError("""The predicted image features are not the same."""	)
				if not np.allclose(snake_case__ ,    snake_case__ ,    atol=1e-3	):
								raise ValueError("""The predicted text features are not the same."""	)
				print("""Model outputs match!"""	)

				if save_model:
								# Create folder to save model
								if not os.path.isdir(snake_case__	):
												os.mkdir(snake_case__	)
								# Save converted model and image processor
								hf_model.save_pretrained(snake_case__	)
								processor.save_pretrained(snake_case__	)

				if push_to_hub:
								# Push model and image processor to hub
								print("""Pushing converted ALIGN to the hub..."""	)
								processor.push_to_hub("""align-base"""	)
								hf_model.push_to_hub("""align-base"""	)


if __name__ == "__main__":
		a								=							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--checkpoint_path',
		    default='./weights/model-weights',
		    type=str,
		    help='Path to the pretrained TF ALIGN checkpoint.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path',
		    default='hf_model',
		    type=str,
		    help='Path to the output PyTorch model directory.',
		)
		parser.add_argument('--save_model', action='store_true', help='Save model to local')
		parser.add_argument('--push_to_hub', action='store_true', help='Push model and image processor to the hub')

		a								=							parser.parse_args()
		convert_align_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub)











"""simple docstring"""







from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a								=							logging.get_logger(__name__)

a								=							{
    'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}






class  SCREAMING_SNAKE_CASE__   (   _a   ):
						_a        =  'mbart'
						_a        =  ['past_key_values']
						_a        =  {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


						def __init__(							self			:     Union[str, Any]  ,  lowerCAmelCase			:     Optional[Any]=5_0265  ,  lowerCAmelCase			:     str=1024  ,  lowerCAmelCase			:     str=12  ,  lowerCAmelCase			:     Any=4096  ,  lowerCAmelCase			:     str=16  ,  lowerCAmelCase			:     Optional[int]=12  ,  lowerCAmelCase			:     Optional[Any]=4096  ,  lowerCAmelCase			:     int=16  ,  lowerCAmelCase			:     Union[str, Any]=0.0  ,  lowerCAmelCase			:     List[str]=0.0  ,  lowerCAmelCase			:     int=True  ,  lowerCAmelCase			:     List[Any]=True  ,  lowerCAmelCase			:     Union[str, Any]="gelu"  ,  lowerCAmelCase			:     List[Any]=1024  ,  lowerCAmelCase			:     Optional[Any]=0.1  ,  lowerCAmelCase			:     List[str]=0.0  ,  lowerCAmelCase			:     str=0.0  ,  lowerCAmelCase			:     str=0.02  ,  lowerCAmelCase			:     int=0.0  ,  lowerCAmelCase			:     int=False  ,  lowerCAmelCase			:     Dict=1  ,  lowerCAmelCase			:     Tuple=0  ,  lowerCAmelCase			:     List[str]=2  ,  lowerCAmelCase			:     Union[str, Any]=2  ,  **lowerCAmelCase			:     Any  ,  ):
										lowerCAmelCase		=			vocab_size
										lowerCAmelCase		=			max_position_embeddings
										lowerCAmelCase		=			d_model
										lowerCAmelCase		=			encoder_ffn_dim
										lowerCAmelCase		=			encoder_layers
										lowerCAmelCase		=			encoder_attention_heads
										lowerCAmelCase		=			decoder_ffn_dim
										lowerCAmelCase		=			decoder_layers
										lowerCAmelCase		=			decoder_attention_heads
										lowerCAmelCase		=			dropout
										lowerCAmelCase		=			attention_dropout
										lowerCAmelCase		=			activation_dropout
										lowerCAmelCase		=			activation_function
										lowerCAmelCase		=			init_std
										lowerCAmelCase		=			encoder_layerdrop
										lowerCAmelCase		=			decoder_layerdrop
										lowerCAmelCase		=			classifier_dropout
										lowerCAmelCase		=			use_cache
										lowerCAmelCase		=			encoder_layers
										lowerCAmelCase		=			scale_embedding  # scale factor will be sqrt(d_model) if True
										super().__init__(
										    pad_token_id=lowerCAmelCase  ,  bos_token_id=lowerCAmelCase  ,  eos_token_id=lowerCAmelCase  ,  is_encoder_decoder=lowerCAmelCase  ,  forced_eos_token_id=lowerCAmelCase  ,  **lowerCAmelCase  ,  )






class  SCREAMING_SNAKE_CASE__   (   _a   ):


						@property
						def 					__lowercase	(							self			:     List[Any]   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														    ]   )

														if self.use_past:
																		lowerCAmelCase		=			{0: """batch"""}
																		lowerCAmelCase		=			{0: """batch""", 1: """past_decoder_sequence + sequence"""}
														else:
																		lowerCAmelCase		=			{0: """batch""", 1: """decoder_sequence"""}
																		lowerCAmelCase		=			{0: """batch""", 1: """decoder_sequence"""}

														if self.use_past:
																		self.fill_with_past_key_values_(lowerCAmelCase  ,  direction="""inputs"""   )
										elif self.task == "causal-lm":
														# TODO: figure this case out.
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														    ]   )
														if self.use_past:
																		lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
																		for i in range(lowerCAmelCase   ):
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
										else:
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
														        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
														    ]   )

										return common_inputs


						@property
						def 					__lowercase	(							self			:     Optional[Any]   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			super().outputs
										else:
														lowerCAmelCase		=			super(lowerCAmelCase  ,  self   ).outputs
														if self.use_past:
																		lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
																		for i in range(lowerCAmelCase   ):
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
										return common_outputs


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )

										# Generate decoder inputs
										lowerCAmelCase		=			seq_length if not self.use_past else 1
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										lowerCAmelCase		=			{f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
										lowerCAmelCase		=			dict(**lowerCAmelCase  ,  **lowerCAmelCase   )

										if self.use_past:
														if not is_torch_available():
																		raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""   )
														else:
																		import torch
														lowerCAmelCase	,  lowerCAmelCase		=			common_inputs["""input_ids"""].shape
														lowerCAmelCase		=			common_inputs["""decoder_input_ids"""].shape[1]
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_attention_heads
														lowerCAmelCase		=			(
														    batch,
														    num_encoder_attention_heads,
														    encoder_seq_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)
														lowerCAmelCase		=			decoder_seq_length + 3
														lowerCAmelCase		=			(
														    batch,
														    num_decoder_attention_heads,
														    decoder_past_length,
														    self._config.hidden_size // num_decoder_attention_heads,
														)

														lowerCAmelCase		=			torch.cat(
														    [common_inputs["""decoder_attention_mask"""], torch.ones(lowerCAmelCase  ,  lowerCAmelCase   )]  ,  dim=1   )

														lowerCAmelCase		=			[]
														# If the number of encoder and decoder layers are present in the model configuration, both are considered
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
														lowerCAmelCase		=			min(lowerCAmelCase  ,  lowerCAmelCase   )
														lowerCAmelCase		=			max(lowerCAmelCase  ,  lowerCAmelCase   ) - min_num_layers
														lowerCAmelCase		=			"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

														for _ in range(lowerCAmelCase   ):
																		common_inputs["past_key_values"].append(
																		    (
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		    )   )
														# TODO: test this.
														lowerCAmelCase		=			encoder_shape if remaining_side_name == """encoder""" else decoder_shape
														for _ in range(lowerCAmelCase  ,  lowerCAmelCase   ):
																		common_inputs["past_key_values"].append((torch.zeros(lowerCAmelCase   ), torch.zeros(lowerCAmelCase   ))   )
										return common_inputs


						def 					__lowercase	(							self			:     Dict  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )

										if self.use_past:
														if not is_torch_available():
																		raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""   )
														else:
																		import torch
														lowerCAmelCase	,  lowerCAmelCase		=			common_inputs["""input_ids"""].shape
														# Not using the same length for past_key_values
														lowerCAmelCase		=			seqlen + 2
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_attention_heads
														lowerCAmelCase		=			(
														    batch,
														    num_encoder_attention_heads,
														    past_key_values_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)

														lowerCAmelCase		=			common_inputs["""attention_mask"""].dtype
														lowerCAmelCase		=			torch.cat(
														    [common_inputs["""attention_mask"""], torch.ones(lowerCAmelCase  ,  lowerCAmelCase  ,  dtype=lowerCAmelCase   )]  ,  dim=1   )
														lowerCAmelCase		=			[
														    (torch.zeros(lowerCAmelCase   ), torch.zeros(lowerCAmelCase   )) for _ in range(lowerCAmelCase   )
														]
										return common_inputs


						def 					__lowercase	(							self			:     str  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										# Copied from OnnxConfig.generate_dummy_inputs
										# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
										# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
										lowerCAmelCase		=			compute_effective_axis_dimension(
										    lowerCAmelCase  ,  fixed_dimension=OnnxConfig.default_fixed_batch  ,  num_token_to_add=0   )

										# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
										lowerCAmelCase		=			tokenizer.num_special_tokens_to_add(lowerCAmelCase   )
										lowerCAmelCase		=			compute_effective_axis_dimension(
										    lowerCAmelCase  ,  fixed_dimension=OnnxConfig.default_fixed_sequence  ,  num_token_to_add=lowerCAmelCase   )

										# Generate dummy inputs according to compute batch and sequence
										lowerCAmelCase		=			[""" """.join([tokenizer.unk_token]   ) * seq_length] * batch_size
										lowerCAmelCase		=			dict(tokenizer(lowerCAmelCase  ,  return_tensors=lowerCAmelCase   )   )
										return common_inputs


						def 					__lowercase	(							self			:     int  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			self._generate_dummy_inputs_for_default_and_seqaseq_lm(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )

										elif self.task == "causal-lm":
														lowerCAmelCase		=			self._generate_dummy_inputs_for_causal_lm(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )
										else:
														lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )

										return common_inputs




						def 					__lowercase	(							self			:     List[str]  ,  lowerCAmelCase			:     Optional[Any]  ,  lowerCAmelCase			:     List[Any]  ,  lowerCAmelCase			:     List[Any]  ,  lowerCAmelCase			:     int   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			super()._flatten_past_key_values_(lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										else:
														lowerCAmelCase		=			super(lowerCAmelCase  ,  self   )._flatten_past_key_values_(
														    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )





