

"""simple docstring"""






import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
					from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
					raise ImportError('''You need to install the version of fairseq from the Mega repo!''')






class       _UpperCAmelCase     (						nn.Module):
			def __init__(						self ,						_A ,						_A ,						_A       )      ->      str:






				'''simple docstring'''







				super().__init__()
				_UpperCAmelCase      :							List[str]       =   mega_args
				_UpperCAmelCase      :							str       =   nn.Embedding(_A ,						self.mega_args.encoder_embed_dim       )
				_UpperCAmelCase      :							Union[str, Any]       =   nn.ModuleList([MegaEncoderLayer(self.mega_args       ) for _ in range(_A       )]       )
				_UpperCAmelCase      :							int       =   depth
			def 							__snake_case	(						self ,						_A ,						_A ,						_A=True ,						_A=0       )      ->      Dict:






				'''simple docstring'''







				if batch_first:
					_UpperCAmelCase      :							Dict       =   input_ids.T

				# to make things more confusing, Mega expects the attention mask to
				# be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
				# which is the opposite of what HF returns
				if ignore_mask_value == 0:
					_UpperCAmelCase      :							Optional[Any]       =   1 - attention_mask

				# get token embeddings from IDs
				_UpperCAmelCase      :							List[Any]       =   self.embedding_layer(_A       )

				# pass through the Mega layers
				# input is (time, batch, encoder dim) and output is the same
				for encoder in self.encoders:
					_UpperCAmelCase      :							Optional[int]       =   encoder(_A ,						_A       )

				# return according to the shape specified
				if batch_first:
					# (T, B, H) --> (B, T, H)
					return torch.transpose(_A ,						0 ,						1       )
				else:
					return embeds






class       _UpperCAmelCase     (						nn.Module):
			def __init__(						self ,						_A ,						_A ,						_A       )      ->      Any:






				'''simple docstring'''







				super().__init__()
				_UpperCAmelCase      :							Dict       =   MegaLM(_A ,						_A ,						_A       )
				_UpperCAmelCase      :							Union[str, Any]       =   nn.Linear(mega_args.encoder_embed_dim ,						_A       )
				_UpperCAmelCase      :							List[Any]       =   nn.Dropout(p=0.1       )
			def 							__snake_case	(						self ,						_A ,						_A ,						_A=True ,						_A=0       )      ->      Optional[int]:






				'''simple docstring'''







				_UpperCAmelCase      :							Any       =   self.mega(_A ,						_A ,						_A ,						_A       )
				return self.mlm_head(self.dropout(_A       )       )
def  UpperCamelCase   (     _lowerCAmelCase       :  Tuple,					_lowerCAmelCase       :  List[str],					_lowerCAmelCase       :  str	)			->     Optional[Any]:





	with open(os.path.join(SCREAMING_SNAKE_CASE_,					"""model_args.pkl"""	),					"""rb"""	) as f:
		_UpperCAmelCase      :							Any       =   pkl.load(SCREAMING_SNAKE_CASE_	)

	# load the original encoder
	_UpperCAmelCase      :							List[str]       =   OriginalMegaForMaskedLM(**SCREAMING_SNAKE_CASE_	).eval()

	# load its weights
	print(
	    """Original Mega encoder:""",					original_mlm.mega.load_state_dict(
	        torch.load(os.path.join(SCREAMING_SNAKE_CASE_,					"""encoder_weights.pt"""	),					map_location="""cpu"""	)	),					)
	print(
	    """Original Mega MLM layer:""",					original_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(SCREAMING_SNAKE_CASE_,					"""mlm_head_weights.pt"""	),					map_location="""cpu"""	)	),					)

	# create a new config from the old one
	_UpperCAmelCase      :							Dict       =   MegaConfig(
	    num_hidden_layers=mega_original_args["""depth"""],					vocab_size=mega_original_args["""vocab_size"""],					hidden_size=mega_original_args["""mega_args"""].encoder_embed_dim,					shared_representation_size=mega_original_args["""mega_args"""].encoder_z_dim,					intermediate_size=mega_original_args["""mega_args"""].encoder_hidden_dim,					ema_projection_size=mega_original_args["""mega_args"""].encoder_n_dim,					dropout_prob=mega_original_args["""mega_args"""].dropout,					attention_probs_dropout_prob=mega_original_args["""mega_args"""].attention_dropout,					hidden_dropout_prob=mega_original_args["""mega_args"""].hidden_dropout,					activation=mega_original_args["""mega_args"""].activation_fn,					attention_activation=mega_original_args["""mega_args"""].attention_activation_fn,					bidirectional=mega_original_args["""mega_args"""].bidirectional,					use_chunking=mega_original_args["""mega_args"""].encoder_chunk_size > 0,					chunk_size=mega_original_args["""mega_args"""].encoder_chunk_size,					truncation=mega_original_args["""mega_args"""].truncation_length,					normalization_type=mega_original_args["""mega_args"""].normalization_type,					normalize_before_mega=SCREAMING_SNAKE_CASE_,					norm_affine=SCREAMING_SNAKE_CASE_,					use_feature_dropout=mega_original_args["""mega_args"""].feature_dropout,					relative_positional_bias=mega_original_args["""mega_args"""].rel_pos_bias,					max_positions=mega_original_args["""mega_args"""].max_source_positions,					nffn_hidden_size=mega_original_args["""mega_args"""].encoder_ffn_embed_dim,					normalize_before_ffn=mega_original_args["""mega_args"""].normalize_before,					nffn_activation_dropout_prob=0.0,					add_token_type_embeddings=SCREAMING_SNAKE_CASE_,					add_lm_hidden_dense_layer=SCREAMING_SNAKE_CASE_,					)

	_UpperCAmelCase      :							Optional[int]       =   MegaForMaskedLM(SCREAMING_SNAKE_CASE_	).eval()

	# the originl checkpoint just uses nn.Embedding for the word embeddings
	# we use a wrapper module for embeddings to add support for positional embeddings
	_UpperCAmelCase      :							Optional[Any]       =   original_mlm.mega.embedding_layer.weight

	# modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
	# ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
	# also renaming previously confusing parameter names
	_UpperCAmelCase      :							Union[str, Any]       =   original_mlm.mega.encoders.state_dict()
	_UpperCAmelCase      :							str       =   {}
	for module_name in original_state_dict.keys():
		_UpperCAmelCase      :							Union[str, Any]       =   None
		# have to handle gamma, beta, and alpha differently due to their use
		# in multiple modules within the original repository;
		# beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
		# the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
		if "beta" in module_name:
			# EMA sub-layers were always called "move" in the original repo
			if "move.beta" in module_name:
				_UpperCAmelCase      :							Optional[int]       =   module_name.replace("""move.beta""",					"""ema_gate.ema_expansion_matrix"""	)
			elif "mega_layer.beta" in module_name:
				_UpperCAmelCase      :							str       =   module_name.replace("""beta""",					"""qk_bias"""	)
			else:
				_UpperCAmelCase      :							Union[str, Any]       =   module_name.replace("""beta""",					"""b_param"""	)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
		elif "gamma" in module_name:
			if "move.gamma" in module_name:
				_UpperCAmelCase      :							Any       =   module_name.replace("""move.gamma""",					"""ema_gate.kernel_projection_matrix"""	)
			elif "mega_layer.gamma" in module_name:
				_UpperCAmelCase      :							Dict       =   module_name.replace("""gamma""",					"""qk_weight"""	)
			else:
				_UpperCAmelCase      :							Optional[Any]       =   module_name.replace("""gamma""",					"""g_param"""	)
        # alpha is used in EMA and positional bias; renaming to improve readability
		elif "move.alpha" in module_name:
			_UpperCAmelCase      :							Tuple       =   module_name.replace("""move.alpha""",					"""ema_gate.decay_factor"""	)
		# delta is only used in EMA; renaming to improve readability
		elif "move.delta" in module_name:
			_UpperCAmelCase      :							List[Any]       =   module_name.replace("""move.delta""",					"""ema_gate.damping_factor"""	)
		# omega is only used in EMA; renaming to improve readability
		elif "omega" in module_name:
			_UpperCAmelCase      :							Union[str, Any]       =   module_name.replace("""move.omega""",					"""ema_gate.residual_weight"""	)

		if new_module_name:
			_UpperCAmelCase      :							Optional[Any]       =   new_module_name

	if len(SCREAMING_SNAKE_CASE_	) != 0:
		print(f'''Renaming these keys: {updated_keys.keys()}'''	)
	else:
		print("""No need to rename state dict entries"""	)
	for old, new in updated_keys.items():
		_UpperCAmelCase      :							Optional[int]       =   original_state_dict.pop(SCREAMING_SNAKE_CASE_	)

	# now attempt to load the state dictionary with updated names
	# note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
	print("""HF Mega encoder:""",					hf_mlm.mega.layers.load_state_dict(SCREAMING_SNAKE_CASE_	)	)

	# load the MLM head weights directly
	print(
	    """HF Mega MLM layer:""",					hf_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(SCREAMING_SNAKE_CASE_,					"""mlm_head_weights.pt"""	),					map_location="""cpu"""	)	),					)

	# test on a randomly generated input sequence
	_UpperCAmelCase      :							List[str]       =   torch.randint(0,					hf_config.vocab_size,					size=(4, 256)	)
	_UpperCAmelCase      :							Union[str, Any]       =   torch.ones_like(SCREAMING_SNAKE_CASE_	)
	# mask a few tokens to make sure masking is applied appropriately :)
	_UpperCAmelCase      :							List[Any]       =   0

	# run forward passes
	_UpperCAmelCase      :							Tuple       =   original_mlm(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					batch_first=SCREAMING_SNAKE_CASE_,					ignore_mask_value=0	)
	_UpperCAmelCase      :							str       =   hf_mlm(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_	)[0]

	# print shapes and diff
	print(f'''original output {original_output.shape}'''	)
	print(f'''hf output {hf_output.shape}'''	)
	print(f'''max diff: {(original_output - hf_output).max()}'''	)  # 0.0
	_UpperCAmelCase      :							List[str]       =   torch.allclose(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					atol=1E-3	)

	if success:
		print("""Yay!"""	)
		hf_mlm.save_pretrained(SCREAMING_SNAKE_CASE_	)
	else:
		raise RuntimeError(f'''Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}'''	)

	if includes_tokenizer:
		print("""Transferring tokenizer"""	)
		_UpperCAmelCase      :							Tuple       =   AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_	)
		tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_	)


if __name__ == "__main__":
					lowerCamelCase__			:					List[Any]							  =					argparse.ArgumentParser()

					parser.add_argument(
					    '''--pretrained_checkpoint_path''',
					    default=None,
					    type=str,
					    required=True,
					    help='''Point to the directory containing your model weights using the official Mega repo''',
					)

					parser.add_argument(
					    '''--output_path''', default=None, type=str, required=True, help='''Location to save the Hugging Face version'''
					)

					parser.add_argument(
					    '''--includes_tokenizer''',
					    action='''store_true''',
					    help='''Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo''',
					)

					lowerCamelCase__			:					Optional[int]							  =					parser.parse_args()

					convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)








"""simple docstring"""






def  UpperCamelCase   (     _lowerCAmelCase       :  float,					_lowerCAmelCase       :  float,					_lowerCAmelCase       :  float	)			->     float:
	if mass < 0:
		raise ValueError("""The mass of the body cannot be negative"""	)
	if radius <= 0:
		raise ValueError("""The radius is always a positive non zero integer"""	)
	return (mass * (velocity) ** 2) / radius


if __name__ == "__main__":
						import doctest

						doctest.testmod(verbose=True)





