




"""simple docstring"""







import argparse
import os

import align
import numpy as np
import requests
import tensorflow as tf
import torch
from PIL import Image
from tokenizer import Tokenizer

from transformers import (
    AlignConfig,
    AlignModel,
    AlignProcessor,
    BertConfig,
    BertTokenizer,
    EfficientNetConfig,
    EfficientNetImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a								=							logging.get_logger(__name__)





def 							lowercase   (snake_case__				:	Union[str, Any]	)       ->					Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			tf.image.resize(snake_case__ ,    (346, 346)	)
				lowerCAmelCase		=			tf.image.crop_to_bounding_box(snake_case__ ,    (346 - 289) // 2 ,    (346 - 289) // 2 ,    289 ,    289	)
				return image





def 							lowercase   ()       ->					Any:


				'''simple docstring'''






				lowerCAmelCase		=			EfficientNetConfig.from_pretrained("""google/efficientnet-b7"""	)
				lowerCAmelCase		=			289
				lowerCAmelCase		=			640
				lowerCAmelCase		=			{"""0""": """LABEL_0""", """1""": """LABEL_1"""}
				lowerCAmelCase		=			{"""LABEL_0""": 0, """LABEL_1""": 1}
				lowerCAmelCase		=			[]

				lowerCAmelCase		=			BertConfig()
				lowerCAmelCase		=			AlignConfig.from_text_vision_configs(
				    text_config=snake_case__ ,    vision_config=snake_case__ ,    projection_dim=640	)
				return config





def 							lowercase   ()       ->					List[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			"""http://images.cocodataset.org/val2017/000000039769.jpg"""
				lowerCAmelCase		=			Image.open(requests.get(snake_case__ ,    stream=snake_case__	).raw	)
				return im





def 							lowercase   ()       ->					Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			EfficientNetImageProcessor(
				    do_center_crop=snake_case__ ,    rescale_factor=1 / 127.5 ,    rescale_offset=snake_case__ ,    do_normalize=snake_case__ ,    include_top=snake_case__ ,    resample=Image.BILINEAR ,    )
				lowerCAmelCase		=			BertTokenizer.from_pretrained("""bert-base-uncased"""	)
				lowerCAmelCase		=			64
				lowerCAmelCase		=			AlignProcessor(image_processor=snake_case__ ,    tokenizer=snake_case__	)
				return processor





def 							lowercase   (snake_case__				:	int	)       ->					str:


				'''simple docstring'''






				lowerCAmelCase		=			[v.split("""_"""	)[0].split("""block"""	)[1] for v in original_param_names if v.startswith("""block"""	)]
				lowerCAmelCase		=			list(set(snake_case__	)	)
				lowerCAmelCase		=			sorted(snake_case__	)
				lowerCAmelCase		=			len(snake_case__	)
				lowerCAmelCase		=			{b: str(snake_case__	) for b, i in zip(snake_case__ ,    range(snake_case__	)	)}

				lowerCAmelCase		=			[]
				rename_keys.append(("""stem_conv/kernel:0""", """embeddings.convolution.weight""")	)
				rename_keys.append(("""stem_bn/gamma:0""", """embeddings.batchnorm.weight""")	)
				rename_keys.append(("""stem_bn/beta:0""", """embeddings.batchnorm.bias""")	)
				rename_keys.append(("""stem_bn/moving_mean:0""", """embeddings.batchnorm.running_mean""")	)
				rename_keys.append(("""stem_bn/moving_variance:0""", """embeddings.batchnorm.running_var""")	)

				for b in block_names:
								lowerCAmelCase		=			block_name_mapping[b]
								rename_keys.append((f'''block{b}_expand_conv/kernel:0''', f'''encoder.blocks.{hf_b}.expansion.expand_conv.weight''')	)
								rename_keys.append((f'''block{b}_expand_bn/gamma:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.weight''')	)
								rename_keys.append((f'''block{b}_expand_bn/beta:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.bias''')	)
								rename_keys.append(
								    (f'''block{b}_expand_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_expand_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.expansion.expand_bn.running_var''')	)
								rename_keys.append(
								    (f'''block{b}_dwconv/depthwise_kernel:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_conv.weight''')	)
								rename_keys.append((f'''block{b}_bn/gamma:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.weight''')	)
								rename_keys.append((f'''block{b}_bn/beta:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.bias''')	)
								rename_keys.append(
								    (f'''block{b}_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_var''')	)

								rename_keys.append((f'''block{b}_se_reduce/kernel:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.reduce.weight''')	)
								rename_keys.append((f'''block{b}_se_reduce/bias:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.reduce.bias''')	)
								rename_keys.append((f'''block{b}_se_expand/kernel:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.expand.weight''')	)
								rename_keys.append((f'''block{b}_se_expand/bias:0''', f'''encoder.blocks.{hf_b}.squeeze_excite.expand.bias''')	)
								rename_keys.append(
								    (f'''block{b}_project_conv/kernel:0''', f'''encoder.blocks.{hf_b}.projection.project_conv.weight''')	)
								rename_keys.append((f'''block{b}_project_bn/gamma:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.weight''')	)
								rename_keys.append((f'''block{b}_project_bn/beta:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.bias''')	)
								rename_keys.append(
								    (f'''block{b}_project_bn/moving_mean:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.running_mean''')	)
								rename_keys.append(
								    (f'''block{b}_project_bn/moving_variance:0''', f'''encoder.blocks.{hf_b}.projection.project_bn.running_var''')	)

				lowerCAmelCase		=			{}
				for item in rename_keys:
								if item[0] in original_param_names:
												lowerCAmelCase		=			"""vision_model.""" + item[1]

    # BERT text encoder
				lowerCAmelCase		=			[]
				lowerCAmelCase		=			"""tf_bert_model/bert"""
				lowerCAmelCase		=			"""text_model"""
				for i in range(12	):
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/query/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.query.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/query/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.query.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/key/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.key.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/key/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.key.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/value/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.value.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/self/value/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.self.value.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/dense/kernel:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.dense.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/dense/bias:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.dense.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/LayerNorm/gamma:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.LayerNorm.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/attention/output/LayerNorm/beta:0''',
								        f'''{new}.encoder.layer.{i}.attention.output.LayerNorm.bias''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/intermediate/dense/kernel:0''',
								        f'''{new}.encoder.layer.{i}.intermediate.dense.weight''',
								    )	)
								rename_keys.append(
								    (
								        f'''{old}/encoder/layer_._{i}/intermediate/dense/bias:0''',
								        f'''{new}.encoder.layer.{i}.intermediate.dense.bias''',
								    )	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/dense/kernel:0''', f'''{new}.encoder.layer.{i}.output.dense.weight''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/dense/bias:0''', f'''{new}.encoder.layer.{i}.output.dense.bias''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/LayerNorm/gamma:0''', f'''{new}.encoder.layer.{i}.output.LayerNorm.weight''')	)
								rename_keys.append(
								    (f'''{old}/encoder/layer_._{i}/output/LayerNorm/beta:0''', f'''{new}.encoder.layer.{i}.output.LayerNorm.bias''')	)

				rename_keys.append((f'''{old}/embeddings/word_embeddings/weight:0''', f'''{new}.embeddings.word_embeddings.weight''')	)
				rename_keys.append(
				    (f'''{old}/embeddings/position_embeddings/embeddings:0''', f'''{new}.embeddings.position_embeddings.weight''')	)
				rename_keys.append(
				    (f'''{old}/embeddings/token_type_embeddings/embeddings:0''', f'''{new}.embeddings.token_type_embeddings.weight''')	)
				rename_keys.append((f'''{old}/embeddings/LayerNorm/gamma:0''', f'''{new}.embeddings.LayerNorm.weight''')	)
				rename_keys.append((f'''{old}/embeddings/LayerNorm/beta:0''', f'''{new}.embeddings.LayerNorm.bias''')	)

				rename_keys.append((f'''{old}/pooler/dense/kernel:0''', f'''{new}.pooler.dense.weight''')	)
				rename_keys.append((f'''{old}/pooler/dense/bias:0''', f'''{new}.pooler.dense.bias''')	)
				rename_keys.append(("""dense/kernel:0""", """text_projection.weight""")	)
				rename_keys.append(("""dense/bias:0""", """text_projection.bias""")	)
				rename_keys.append(("""dense/bias:0""", """text_projection.bias""")	)
				rename_keys.append(("""temperature:0""", """temperature""")	)

				for item in rename_keys:
								if item[0] in original_param_names:
												lowerCAmelCase		=			item[1]
				return key_mapping





def 							lowercase   (snake_case__				:	int ,    snake_case__				:	List[Any] ,    snake_case__				:	Any	)       ->					List[Any]:


				'''simple docstring'''






				list(hf_params.keys()	)

				for key, value in tf_params.items():
								if key not in key_mapping:
												continue

								lowerCAmelCase		=			key_mapping[key]
								if "_conv" in key and "kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	).permute(3 ,    2 ,    0 ,    1	)
								elif "embeddings" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	)
								elif "depthwise_kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	).permute(2 ,    3 ,    0 ,    1	)
								elif "kernel" in key:
												lowerCAmelCase		=			torch.from_numpy(np.transpose(snake_case__	)	)
								elif "temperature" in key:
												lowerCAmelCase		=			value
								elif "bn/gamma" or "bn/beta" in key:
												lowerCAmelCase		=			torch.from_numpy(np.transpose(snake_case__	)	).squeeze()
								else:
												lowerCAmelCase		=			torch.from_numpy(snake_case__	)

								# Replace HF parameters with original TF model parameters
								hf_params[hf_key].copy_(snake_case__	)





@torch.no_grad()
def 							lowercase   (snake_case__				:	Dict ,    snake_case__				:	Dict ,    snake_case__				:	str ,    snake_case__				:	Union[str, Any]	)       ->					List[str]:


				'''simple docstring'''






				lowerCAmelCase		=			64
				lowerCAmelCase		=			Tokenizer(snake_case__	)
				lowerCAmelCase		=			align.Align("""efficientnet-b7""" ,    """bert-base""" ,    640 ,    snake_case__ ,    tok.get_vocab_size()	)
				original_model.compile()
				original_model.load_weights(snake_case__	)

				lowerCAmelCase		=			original_model.trainable_variables
				lowerCAmelCase		=			original_model.non_trainable_variables
				lowerCAmelCase		=			{param.name: param.numpy() for param in tf_params}
				for param in tf_non_train_params:
								lowerCAmelCase		=			param.numpy()
				lowerCAmelCase		=			list(tf_params.keys()	)

				# Load HuggingFace model
				lowerCAmelCase		=			get_align_config()
				lowerCAmelCase		=			AlignModel(snake_case__	).eval()
				lowerCAmelCase		=			hf_model.state_dict()

				# Create src-to-dst parameter name mapping dictionary
				print("""Converting parameters..."""	)
				lowerCAmelCase		=			rename_keys(snake_case__	)
				replace_params(snake_case__ ,    snake_case__ ,    snake_case__	)

				# Initialize processor
				lowerCAmelCase		=			get_processor()
				lowerCAmelCase		=			processor(
				    images=prepare_img() ,    text="""A picture of a cat""" ,    padding="""max_length""" ,    max_length=64 ,    return_tensors="""pt"""	)

				# HF model inference
				hf_model.eval()
				with torch.no_grad():
								lowerCAmelCase		=			hf_model(**snake_case__	)

				lowerCAmelCase		=			outputs.image_embeds.detach().numpy()
				lowerCAmelCase		=			outputs.text_embeds.detach().numpy()

				# Original model inference
				lowerCAmelCase		=			False
				lowerCAmelCase		=			EfficientNetImageProcessor(
				    do_center_crop=snake_case__ ,    do_rescale=snake_case__ ,    do_normalize=snake_case__ ,    include_top=snake_case__ ,    resample=Image.BILINEAR ,    )
				lowerCAmelCase		=			tf_image_processor(images=prepare_img() ,    return_tensors="""tf""" ,    data_format="""channels_last"""	)["""pixel_values"""]
				lowerCAmelCase		=			tok(tf.constant(["""A picture of a cat"""]	)	)

				lowerCAmelCase		=			original_model.image_encoder(snake_case__ ,    training=snake_case__	)
				lowerCAmelCase		=			original_model.text_encoder(snake_case__ ,    training=snake_case__	)

				lowerCAmelCase		=			tf.nn.la_normalize(snake_case__ ,    axis=-1	)
				lowerCAmelCase		=			tf.nn.la_normalize(snake_case__ ,    axis=-1	)

				# Check whether original and HF model outputs match  -> np.allclose
				if not np.allclose(snake_case__ ,    snake_case__ ,    atol=1e-3	):
								raise ValueError("""The predicted image features are not the same."""	)
				if not np.allclose(snake_case__ ,    snake_case__ ,    atol=1e-3	):
								raise ValueError("""The predicted text features are not the same."""	)
				print("""Model outputs match!"""	)

				if save_model:
								# Create folder to save model
								if not os.path.isdir(snake_case__	):
												os.mkdir(snake_case__	)
								# Save converted model and image processor
								hf_model.save_pretrained(snake_case__	)
								processor.save_pretrained(snake_case__	)

				if push_to_hub:
								# Push model and image processor to hub
								print("""Pushing converted ALIGN to the hub..."""	)
								processor.push_to_hub("""align-base"""	)
								hf_model.push_to_hub("""align-base"""	)


if __name__ == "__main__":
		a								=							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--checkpoint_path',
		    default='./weights/model-weights',
		    type=str,
		    help='Path to the pretrained TF ALIGN checkpoint.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path',
		    default='hf_model',
		    type=str,
		    help='Path to the output PyTorch model directory.',
		)
		parser.add_argument('--save_model', action='store_true', help='Save model to local')
		parser.add_argument('--push_to_hub', action='store_true', help='Push model and image processor to the hub')

		a								=							parser.parse_args()
		convert_align_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub)











"""simple docstring"""







from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a								=							logging.get_logger(__name__)

a								=							{
    'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}






class  SCREAMING_SNAKE_CASE__   (   _a   ):
						_a        =  'mbart'
						_a        =  ['past_key_values']
						_a        =  {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


						def __init__(							self			:     Union[str, Any]  ,  lowerCAmelCase			:     Optional[Any]=5_0265  ,  lowerCAmelCase			:     str=1024  ,  lowerCAmelCase			:     str=12  ,  lowerCAmelCase			:     Any=4096  ,  lowerCAmelCase			:     str=16  ,  lowerCAmelCase			:     Optional[int]=12  ,  lowerCAmelCase			:     Optional[Any]=4096  ,  lowerCAmelCase			:     int=16  ,  lowerCAmelCase			:     Union[str, Any]=0.0  ,  lowerCAmelCase			:     List[str]=0.0  ,  lowerCAmelCase			:     int=True  ,  lowerCAmelCase			:     List[Any]=True  ,  lowerCAmelCase			:     Union[str, Any]="gelu"  ,  lowerCAmelCase			:     List[Any]=1024  ,  lowerCAmelCase			:     Optional[Any]=0.1  ,  lowerCAmelCase			:     List[str]=0.0  ,  lowerCAmelCase			:     str=0.0  ,  lowerCAmelCase			:     str=0.02  ,  lowerCAmelCase			:     int=0.0  ,  lowerCAmelCase			:     int=False  ,  lowerCAmelCase			:     Dict=1  ,  lowerCAmelCase			:     Tuple=0  ,  lowerCAmelCase			:     List[str]=2  ,  lowerCAmelCase			:     Union[str, Any]=2  ,  **lowerCAmelCase			:     Any  ,  ):
										lowerCAmelCase		=			vocab_size
										lowerCAmelCase		=			max_position_embeddings
										lowerCAmelCase		=			d_model
										lowerCAmelCase		=			encoder_ffn_dim
										lowerCAmelCase		=			encoder_layers
										lowerCAmelCase		=			encoder_attention_heads
										lowerCAmelCase		=			decoder_ffn_dim
										lowerCAmelCase		=			decoder_layers
										lowerCAmelCase		=			decoder_attention_heads
										lowerCAmelCase		=			dropout
										lowerCAmelCase		=			attention_dropout
										lowerCAmelCase		=			activation_dropout
										lowerCAmelCase		=			activation_function
										lowerCAmelCase		=			init_std
										lowerCAmelCase		=			encoder_layerdrop
										lowerCAmelCase		=			decoder_layerdrop
										lowerCAmelCase		=			classifier_dropout
										lowerCAmelCase		=			use_cache
										lowerCAmelCase		=			encoder_layers
										lowerCAmelCase		=			scale_embedding  # scale factor will be sqrt(d_model) if True
										super().__init__(
										    pad_token_id=lowerCAmelCase  ,  bos_token_id=lowerCAmelCase  ,  eos_token_id=lowerCAmelCase  ,  is_encoder_decoder=lowerCAmelCase  ,  forced_eos_token_id=lowerCAmelCase  ,  **lowerCAmelCase  ,  )






class  SCREAMING_SNAKE_CASE__   (   _a   ):


						@property
						def 					__lowercase	(							self			:     List[Any]   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														    ]   )

														if self.use_past:
																		lowerCAmelCase		=			{0: """batch"""}
																		lowerCAmelCase		=			{0: """batch""", 1: """past_decoder_sequence + sequence"""}
														else:
																		lowerCAmelCase		=			{0: """batch""", 1: """decoder_sequence"""}
																		lowerCAmelCase		=			{0: """batch""", 1: """decoder_sequence"""}

														if self.use_past:
																		self.fill_with_past_key_values_(lowerCAmelCase  ,  direction="""inputs"""   )
										elif self.task == "causal-lm":
														# TODO: figure this case out.
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														    ]   )
														if self.use_past:
																		lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
																		for i in range(lowerCAmelCase   ):
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
										else:
														lowerCAmelCase		=			OrderedDict(
														    [
														        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
														        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
														        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
														    ]   )

										return common_inputs


						@property
						def 					__lowercase	(							self			:     Optional[Any]   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			super().outputs
										else:
														lowerCAmelCase		=			super(lowerCAmelCase  ,  self   ).outputs
														if self.use_past:
																		lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
																		for i in range(lowerCAmelCase   ):
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
																						lowerCAmelCase		=			{0: """batch""", 2: """past_sequence + sequence"""}
										return common_outputs


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )

										# Generate decoder inputs
										lowerCAmelCase		=			seq_length if not self.use_past else 1
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										lowerCAmelCase		=			{f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
										lowerCAmelCase		=			dict(**lowerCAmelCase  ,  **lowerCAmelCase   )

										if self.use_past:
														if not is_torch_available():
																		raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""   )
														else:
																		import torch
														lowerCAmelCase	,  lowerCAmelCase		=			common_inputs["""input_ids"""].shape
														lowerCAmelCase		=			common_inputs["""decoder_input_ids"""].shape[1]
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_attention_heads
														lowerCAmelCase		=			(
														    batch,
														    num_encoder_attention_heads,
														    encoder_seq_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)
														lowerCAmelCase		=			decoder_seq_length + 3
														lowerCAmelCase		=			(
														    batch,
														    num_decoder_attention_heads,
														    decoder_past_length,
														    self._config.hidden_size // num_decoder_attention_heads,
														)

														lowerCAmelCase		=			torch.cat(
														    [common_inputs["""decoder_attention_mask"""], torch.ones(lowerCAmelCase  ,  lowerCAmelCase   )]  ,  dim=1   )

														lowerCAmelCase		=			[]
														# If the number of encoder and decoder layers are present in the model configuration, both are considered
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
														lowerCAmelCase		=			min(lowerCAmelCase  ,  lowerCAmelCase   )
														lowerCAmelCase		=			max(lowerCAmelCase  ,  lowerCAmelCase   ) - min_num_layers
														lowerCAmelCase		=			"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

														for _ in range(lowerCAmelCase   ):
																		common_inputs["past_key_values"].append(
																		    (
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		        torch.zeros(lowerCAmelCase   ),
																		    )   )
														# TODO: test this.
														lowerCAmelCase		=			encoder_shape if remaining_side_name == """encoder""" else decoder_shape
														for _ in range(lowerCAmelCase  ,  lowerCAmelCase   ):
																		common_inputs["past_key_values"].append((torch.zeros(lowerCAmelCase   ), torch.zeros(lowerCAmelCase   ))   )
										return common_inputs


						def 					__lowercase	(							self			:     Dict  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )

										if self.use_past:
														if not is_torch_available():
																		raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""   )
														else:
																		import torch
														lowerCAmelCase	,  lowerCAmelCase		=			common_inputs["""input_ids"""].shape
														# Not using the same length for past_key_values
														lowerCAmelCase		=			seqlen + 2
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_layers
														lowerCAmelCase	,  lowerCAmelCase		=			self.num_attention_heads
														lowerCAmelCase		=			(
														    batch,
														    num_encoder_attention_heads,
														    past_key_values_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)

														lowerCAmelCase		=			common_inputs["""attention_mask"""].dtype
														lowerCAmelCase		=			torch.cat(
														    [common_inputs["""attention_mask"""], torch.ones(lowerCAmelCase  ,  lowerCAmelCase  ,  dtype=lowerCAmelCase   )]  ,  dim=1   )
														lowerCAmelCase		=			[
														    (torch.zeros(lowerCAmelCase   ), torch.zeros(lowerCAmelCase   )) for _ in range(lowerCAmelCase   )
														]
										return common_inputs


						def 					__lowercase	(							self			:     str  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										# Copied from OnnxConfig.generate_dummy_inputs
										# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
										# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
										lowerCAmelCase		=			compute_effective_axis_dimension(
										    lowerCAmelCase  ,  fixed_dimension=OnnxConfig.default_fixed_batch  ,  num_token_to_add=0   )

										# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
										lowerCAmelCase		=			tokenizer.num_special_tokens_to_add(lowerCAmelCase   )
										lowerCAmelCase		=			compute_effective_axis_dimension(
										    lowerCAmelCase  ,  fixed_dimension=OnnxConfig.default_fixed_sequence  ,  num_token_to_add=lowerCAmelCase   )

										# Generate dummy inputs according to compute batch and sequence
										lowerCAmelCase		=			[""" """.join([tokenizer.unk_token]   ) * seq_length] * batch_size
										lowerCAmelCase		=			dict(tokenizer(lowerCAmelCase  ,  return_tensors=lowerCAmelCase   )   )
										return common_inputs


						def 					__lowercase	(							self			:     int  ,  lowerCAmelCase			:     PreTrainedTokenizer  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     int = -1  ,  lowerCAmelCase			:     bool = False  ,  lowerCAmelCase			:     Optional[TensorType] = None  ,  ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			self._generate_dummy_inputs_for_default_and_seqaseq_lm(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )

										elif self.task == "causal-lm":
														lowerCAmelCase		=			self._generate_dummy_inputs_for_causal_lm(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )
										else:
														lowerCAmelCase		=			self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
														    lowerCAmelCase  ,  batch_size=lowerCAmelCase  ,  seq_length=lowerCAmelCase  ,  is_pair=lowerCAmelCase  ,  framework=lowerCAmelCase   )

										return common_inputs




						def 					__lowercase	(							self			:     List[str]  ,  lowerCAmelCase			:     Optional[Any]  ,  lowerCAmelCase			:     List[Any]  ,  lowerCAmelCase			:     List[Any]  ,  lowerCAmelCase			:     int   ):
										if self.task in ["default", "seq2seq-lm"]:
														lowerCAmelCase		=			super()._flatten_past_key_values_(lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										else:
														lowerCAmelCase		=			super(lowerCAmelCase  ,  self   )._flatten_past_key_values_(
														    lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )





