
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig

def 					lowerCAmelCase__  (							_SCREAMING_SNAKE_CASE    :     List[Any]			,       _SCREAMING_SNAKE_CASE    :     str			,       _SCREAMING_SNAKE_CASE    :     Dict=0      ):
				"""simple docstring"""







				if name is None:
								__a					       =						None
				else:
								__a					       =						'.' * max(0			,       spaces - 2      ) + '# {:' + str(50 - spaces      ) + 's}'
								__a					       =						fmt.format(SCREAMING_SNAKE_CASE__      )

				# Print and recurse (if needed).
				if isinstance(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      ):
								if msg is not None:
												print(SCREAMING_SNAKE_CASE__      )
								for k in val.keys():
												recursive_print(SCREAMING_SNAKE_CASE__			,       val[k]			,       spaces + 2      )
				elif isinstance(SCREAMING_SNAKE_CASE__			,       torch.Tensor      ):
								print(SCREAMING_SNAKE_CASE__			,       """:"""			,       val.size()      )
				else:
								print(SCREAMING_SNAKE_CASE__			,       """:"""			,       SCREAMING_SNAKE_CASE__      )

def 					lowerCAmelCase__  (							_SCREAMING_SNAKE_CASE    :     Tuple			,       _SCREAMING_SNAKE_CASE    :     str			,       _SCREAMING_SNAKE_CASE    :     str			,       _SCREAMING_SNAKE_CASE    :     Optional[int]			,       _SCREAMING_SNAKE_CASE    :     Union[str, Any]      ):
				"""simple docstring"""







				__a					       =						param.size()
				if checkpoint_version == 1.0:
								# version 1.0 stores [num_heads * hidden_size * num_splits, :]
								__a					       =						(num_heads, hidden_size, num_splits) + input_shape[1:]
								__a					       =						param.view(*SCREAMING_SNAKE_CASE__      )
								__a					       =						param.transpose(0			,       2      )
								__a					       =						param.transpose(1			,       2      ).contiguous()
				elif checkpoint_version >= 2.0:
								# other versions store [num_heads * num_splits * hidden_size, :]
								__a					       =						(num_heads, num_splits, hidden_size) + input_shape[1:]
								__a					       =						param.view(*SCREAMING_SNAKE_CASE__      )
								__a					       =						param.transpose(0			,       1      ).contiguous()
				__a					       =						param.view(*SCREAMING_SNAKE_CASE__      )
				return param

def 					lowerCAmelCase__  (							_SCREAMING_SNAKE_CASE    :     Union[str, Any]			,       _SCREAMING_SNAKE_CASE    :     Dict			,       _SCREAMING_SNAKE_CASE    :     Any      ):
				"""simple docstring"""







				__a					       =						{}

				# old versions did not store training args
				__a					       =						input_state_dict.get("""args"""			,       SCREAMING_SNAKE_CASE__      )
				if ds_args is not None:
								# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
								# from pprint import pprint
								# pprint(vars(ds_args))

								__a					       =						ds_args.tokenizer_type
								__a					       =						ds_args.padded_vocab_size
								__a					       =						ds_args.max_position_embeddings
								__a					       =						ds_args.hidden_size
								__a					       =						ds_args.num_layers
								__a					       =						ds_args.num_attention_heads
								__a					       =						ds_args.ffn_hidden_size if 'ffn_hidden_size' in ds_args else 4 * ds_args.hidden_size
								# pprint(config)

				# The number of heads.
				__a					       =						config.num_attention_heads
				# The hidden_size per head.
				__a					       =						config.hidden_size // heads
				# Megatron-LM checkpoint version
				if "checkpoint_version" in input_state_dict.keys():
								__a					       =						input_state_dict['checkpoint_version']
				else:
								__a					       =						0.0

				# The model.
				__a					       =						input_state_dict['model']
				# The language model.
				__a					       =						model['language_model']
				# The embeddings.
				__a					       =						lm['embedding']

				# The word embeddings.
				__a					       =						embeddings['word_embeddings']['weight']
				# Truncate the embedding table to vocab_size rows.
				__a					       =						word_embeddings[: config.vocab_size, :]
				# Store the word embeddings.
				__a					       =						word_embeddings

				# The position embeddings.
				__a					       =						embeddings['position_embeddings']['weight']
				assert pos_embeddings.size(0      ) == config.max_position_embeddings and pos_embeddings.size(1      ) == config.hidden_size
				# Store the position embeddings.
				__a					       =						pos_embeddings

				# The token-type embeddings.
				__a					       =						embeddings['tokentype_embeddings']['weight']
				# Store the position embeddings.
				__a					       =						tokentype_embeddings

				# The transformer.
				__a					       =						lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

				# The regex to extract layer names.
				__a					       =						re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""      )

				# The simple map of names for "automated" rules.
				__a					       =						{
				    'attention.dense': '.attention.output.dense.',
				    'self_attention.dense': '.attention.output.dense.',
				    'mlp.dense_h_to_4h': '.intermediate.dense.',
				    'mlp.dense_4h_to_h': '.output.dense.',
				}

				# Keep track of the attention/query/value tensor.
				__a					       =						None

				# Extract the layers.
				for key, val in transformer.items():
								# Match the name.
								__a					       =						layer_re.match(SCREAMING_SNAKE_CASE__      )

								# Stop if that's not a layer
								if m is None:
												break

								# The index of the layer.
								__a					       =						int(m.group(1      )      )
								# The name of the operation.
								__a					       =						m.group(2      )
								# Is it a weight or a bias?
								__a					       =						m.group(3      )

								# The name of the layer.
								__a					       =						f"bert.encoder.layer.{layer_idx}"

								# For layernorm(s), simply store the layer norm.
								if op_name.endswith("""layernorm"""      ):
												__a					       =						'attention.ln' if op_name.startswith("""input"""      ) else 'ln'
												__a					       =						val

								# Transpose the QKV matrix.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "weight":
												# Make sure the QKV pointer is nil.
												assert attention_qkv_weight is None, ""

												__a					       =						fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__			,       3			,       SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      )
												# Store the tensor as we need the bias as well to interleave QKV and biases.
												__a					       =						out_val

								# Transpose the bias.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "bias":
												# Make sure we read the weight tensor.
												assert attention_qkv_weight is not None, ""

												# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
												__a					       =						attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
												__a					       =						attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
												__a					       =						attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

												__a					       =						fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__			,       3			,       SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      )
												# Split the bias.
												__a					       =						out_val[0 * config.hidden_size : 1 * config.hidden_size]
												__a					       =						out_val[1 * config.hidden_size : 2 * config.hidden_size]
												__a					       =						out_val[2 * config.hidden_size : 3 * config.hidden_size]

												# Store.
												__a					       =						q
												__a					       =						q_bias
												__a					       =						k
												__a					       =						k_bias
												__a					       =						v
												__a					       =						v_bias

												# Clear the stored tensor.
												__a					       =						None

								# Copy weights and biases as is.
								elif weight_or_bias in ["weight", "bias"]:
												__a					       =						megatron_to_transformers[op_name]
												__a					       =						val

    # The final layernorm.
				__a					       =						transformer['final_layernorm.weight']
				__a					       =						transformer['final_layernorm.bias']

				# The pooler.
				__a					       =						lm['pooler']

				# Store the matrix and the bias.
				__a					       =						pooler['dense.weight']
				__a					       =						pooler['dense.bias']

				# The LM head from Megatron (for RACE).
				__a					       =						model['lm_head']

				# The transform matrix.
				__a					       =						lm_head['dense.weight']
				__a					       =						lm_head['dense.bias']

				# The transform LN.
				__a					       =						lm_head['layernorm.weight']
				__a					       =						lm_head['layernorm.bias']

				# For the decoder, we replicate the weights.
				__a					       =						word_embeddings
				__a					       =						lm_head['bias']

				# The classifier from Megatron (for MLNI).
				__a					       =						model['binary_head']

				# Store the classifier.
				__a					       =						binary_head['weight']
				__a					       =						binary_head['bias']

				# It should be done!
				return output_state_dict

def 					lowerCAmelCase__  (							):
				"""simple docstring"""







				__a					       =						argparse.ArgumentParser()
				parser.add_argument("""--print-checkpoint-structure"""			,       action="""store_true"""      )
				parser.add_argument("""path_to_checkpoint"""			,       type=SCREAMING_SNAKE_CASE__			,       help="""Path to the ZIP file containing the checkpoint"""      )
				parser.add_argument(
				    """--config_file"""			,       default=""""""			,       type=SCREAMING_SNAKE_CASE__			,       help="""An optional config json file describing the pre-trained model."""			,       )
				__a					       =						parser.parse_args()

				# Extract the basename.
				__a					       =						os.path.dirname(args.path_to_checkpoint      )

				# Load the model.
				# the .zip is very optional, let's keep it for backward compatibility
				print(f"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""      )
				if args.path_to_checkpoint.endswith(""".zip"""      ):
								with zipfile.ZipFile(args.path_to_checkpoint			,       """r"""      ) as checkpoint:
												with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""      ) as pytorch_dict:
																__a					       =						torch.load(SCREAMING_SNAKE_CASE__			,       map_location="""cpu"""      )
				else:
								__a					       =						torch.load(args.path_to_checkpoint			,       map_location="""cpu"""      )

				if args.config_file == "":
								# Default config of megatron-bert 345m
								__a					       =						MegatronBertConfig()

								# different megatron-bert-*-345m models have different vocab sizes, so override the default
								# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
								__a					       =						input_state_dict['model']['lm_head']['bias'].numel()
				else:
								__a					       =						MegatronBertConfig.from_json_file(args.config_file      )

				# Convert.
				print("""Converting"""      )
				__a					       =						convert_megatron_checkpoint(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      )

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      )

				# Store the config to file.
				print("""Saving config"""      )
				config.save_pretrained(SCREAMING_SNAKE_CASE__      )

				# Store the state_dict to file.
				__a					       =						os.path.join(SCREAMING_SNAKE_CASE__			,       """pytorch_model.bin"""      )
				print(f"Saving checkpoint to \"{output_checkpoint_file}\""      )
				torch.save(SCREAMING_SNAKE_CASE__			,       SCREAMING_SNAKE_CASE__      )


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################





import inspect
import unittest

from transformers import ResNetConfig, is_flax_available
from transformers.testing_utils import require_flax, slow
from transformers.utils import cached_property, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
						import jax
						import jax.numpy as jnp

						from transformers.models.resnet.modeling_flax_resnet import FlaxResNetForImageClassification, FlaxResNetModel

if is_vision_available():
						from PIL import Image

						from transformers import AutoImageProcessor






class 				SCREAMING_SNAKE_CASE     (							unittest.TestCase   ):





							def __init__(				self						:						Optional[int] ,			__lowercase						:						str ,			__lowercase						:						str=3 ,			__lowercase						:						Optional[int]=32 ,			__lowercase						:						Union[str, Any]=3 ,			__lowercase						:						str=10 ,			__lowercase						:						int=[10, 20, 30, 40] ,			__lowercase						:						Any=[1, 1, 2, 1] ,			__lowercase						:						Optional[int]=True ,			__lowercase						:						Optional[int]=True ,			__lowercase						:						int="relu" ,			__lowercase						:						Union[str, Any]=3 ,			__lowercase						:						List[str]=None ,			):

											'''simple docstring'''
											__a					       =						parent
											__a					       =						batch_size
											__a					       =						image_size
											__a					       =						num_channels
											__a					       =						embeddings_size
											__a					       =						hidden_sizes
											__a					       =						depths
											__a					       =						is_training
											__a					       =						use_labels
											__a					       =						hidden_act
											__a					       =						num_labels
											__a					       =						scope
											__a					       =						len(__lowercase		)





							def   UpperCamelCase_   (				self						:						List[Any]		):

											'''simple docstring'''
											__a					       =						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

											__a					       =						self.get_config()

											return config, pixel_values





							def   UpperCamelCase_   (				self						:						Optional[int]		):

											'''simple docstring'''
											return ResNetConfig(
											    num_channels=self.num_channels ,			embeddings_size=self.embeddings_size ,			hidden_sizes=self.hidden_sizes ,			depths=self.depths ,			hidden_act=self.hidden_act ,			num_labels=self.num_labels ,			image_size=self.image_size ,			)





							def   UpperCamelCase_   (				self						:						List[str] ,			__lowercase						:						Union[str, Any] ,			__lowercase						:						str		):

											'''simple docstring'''
											__a					       =						FlaxResNetModel(config=__lowercase		)
											__a					       =						model(__lowercase		)

											# Output shape (b, c, h, w)
											self.parent.assertEqual(
											    result.last_hidden_state.shape ,			(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32) ,			)





							def   UpperCamelCase_   (				self						:						List[str] ,			__lowercase						:						Dict ,			__lowercase						:						Optional[int]		):

											'''simple docstring'''
											__a					       =						self.num_labels
											__a					       =						FlaxResNetForImageClassification(config=__lowercase		)
											__a					       =						model(__lowercase		)
											self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.num_labels)		)







							def   UpperCamelCase_   (				self						:						List[Any]		):

											'''simple docstring'''
											__a					       =						self.prepare_config_and_inputs()
											__a       ,   __a					       =						config_and_inputs
											__a					       =						{"""pixel_values""": pixel_values}
											return config, inputs_dict






@require_flax
class 				SCREAMING_SNAKE_CASE     (							lowerCamelCase__     , unittest.TestCase   ):
							__lowerCamelCase		:		int      =(FlaxResNetModel, FlaxResNetForImageClassification) if is_flax_available() else ()

							__lowerCamelCase		:		Union[str, Any]      =False
							__lowerCamelCase		:		Optional[Any]      =False
							__lowerCamelCase		:		str      =False





							def   UpperCamelCase_   (				self						:						Optional[Any]		):

											'''simple docstring'''
											__a					       =						FlaxResNetModelTester(self		)
											__a					       =						ConfigTester(self ,			config_class=__lowercase ,			has_text_modality=__lowercase		)





							def   UpperCamelCase_   (				self						:						str		):

											'''simple docstring'''
											self.create_and_test_config_common_properties()
											self.config_tester.create_and_test_config_to_json_string()
											self.config_tester.create_and_test_config_to_json_file()
											self.config_tester.create_and_test_config_from_and_save_pretrained()
											self.config_tester.create_and_test_config_with_num_labels()
											self.config_tester.check_config_can_be_init_without_params()
											self.config_tester.check_config_arguments_init()





							def   UpperCamelCase_   (				self						:						str		):

											'''simple docstring'''
											return





							def   UpperCamelCase_   (				self						:						Any		):

											'''simple docstring'''
											__a					       =						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__lowercase		)





							def   UpperCamelCase_   (				self						:						Dict		):

											'''simple docstring'''
											__a					       =						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_image_classification(*__lowercase		)





							@unittest.skip(reason="""ResNet does not use inputs_embeds"""		)
							def   UpperCamelCase_   (				self						:						Optional[Any]		):

											'''simple docstring'''
											pass





							@unittest.skip(reason="""ResNet does not support input and output embeddings"""		)
							def   UpperCamelCase_   (				self						:						int		):

											'''simple docstring'''
											pass





							def   UpperCamelCase_   (				self						:						int		):

											'''simple docstring'''
											__a       ,   __a					       =						self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															__a					       =						model_class(__lowercase		)
															__a					       =						inspect.signature(model.__call__		)
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															__a					       =						[*signature.parameters.keys()]

															__a					       =						["""pixel_values"""]
															self.assertListEqual(arg_names[:1] ,			__lowercase		)





							def   UpperCamelCase_   (				self						:						Union[str, Any]		):

											'''simple docstring'''
											def check_hidden_states_output(__lowercase						:						Tuple ,			__lowercase						:						List[str] ,			__lowercase						:						str		):
															__a					       =						model_class(__lowercase		)

															__a					       =						model(**self._prepare_for_class(__lowercase ,			__lowercase		)		)

															__a					       =						outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

															__a					       =						self.model_tester.num_stages
															self.assertEqual(len(__lowercase		) ,			expected_num_stages + 1		)





							@unittest.skip(reason="""ResNet does not use feedforward chunking"""		)
							def   UpperCamelCase_   (				self						:						Optional[Any]		):

											'''simple docstring'''
											pass







							def   UpperCamelCase_   (				self						:						List[Any]		):

											'''simple docstring'''
											__a       ,   __a					       =						self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															with self.subTest(model_class.__name__		):
																			__a					       =						self._prepare_for_class(__lowercase ,			__lowercase		)
																			__a					       =						model_class(__lowercase		)

																			@jax.jit
																			def model_jitted(__lowercase						:						List[str] ,			**__lowercase						:						Optional[int]		):
																							return model(pixel_values=__lowercase ,			**__lowercase		)

																			with self.subTest("""JIT Enabled"""		):
																							__a					       =						model_jitted(**__lowercase		).to_tuple()

																			with self.subTest("""JIT Disabled"""		):
																							with jax.disable_jit():
																											__a					       =						model_jitted(**__lowercase		).to_tuple()

																			self.assertEqual(len(__lowercase		) ,			len(__lowercase		)		)
																			for jitted_output, output in zip(__lowercase ,			__lowercase		):
																							self.assertEqual(jitted_output.shape ,			output.shape		)



def 					lowerCAmelCase__  (							):
				"""simple docstring"""







				__a					       =						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""      )
				return image






@require_flax
class 				SCREAMING_SNAKE_CASE     (							unittest.TestCase   ):





							@cached_property
							def   UpperCamelCase_   (				self						:						Union[str, Any]		):

											'''simple docstring'''
											return AutoImageProcessor.from_pretrained("""microsoft/resnet-50"""		) if is_vision_available() else None







							@slow
							def   UpperCamelCase_   (				self						:						int		):

											'''simple docstring'''
											__a					       =						FlaxResNetForImageClassification.from_pretrained("""microsoft/resnet-50"""		)

											__a					       =						self.default_image_processor
											__a					       =						prepare_img()
											__a					       =						image_processor(images=__lowercase ,			return_tensors="""np"""		)

											__a					       =						model(**__lowercase		)

											# verify the logits
											__a					       =						(1, 1000)
											self.assertEqual(outputs.logits.shape ,			__lowercase		)

											__a					       =						jnp.array([-11.1069, -9.7877, -8.3777]		)

											self.assertTrue(jnp.allclose(outputs.logits[0, :3] ,			__lowercase ,			atol=1E-4		)		)
