




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def 							__SCREAMING_SNAKE_CASE  (      snake_case_     ,   snake_case_     ,   snake_case_=0						):

						'''simple docstring'''


						if name is None:
												_UpperCAmelCase													=					None
						else:
												_UpperCAmelCase													=					"." * max(0     ,   spaces - 2						) + "# {:" + str(50 - spaces						) + "s}"
												_UpperCAmelCase													=					fmt.format(snake_case_						)

						# Print and recurse (if needed).
						if isinstance(snake_case_     ,   snake_case_						):
												if msg is not None:
																		print(snake_case_						)
												for k in val.keys():
																		recursive_print(snake_case_     ,   val[k]     ,   spaces + 2						)
						elif isinstance(snake_case_     ,   torch.Tensor						):
												print(snake_case_     ,   ":"     ,   val.size()						)
						else:
												print(snake_case_     ,   ":"     ,   snake_case_						)


def 							__SCREAMING_SNAKE_CASE  (      snake_case_     ,   snake_case_     ,   snake_case_     ,   snake_case_						):

						'''simple docstring'''


						_UpperCAmelCase													=					param.size()
						# other versions store [num_heads * num_splits * hidden_size, :]
						_UpperCAmelCase													=					(num_heads, num_splits, hidden_size) + input_shape[1:]
						_UpperCAmelCase													=					param.view(*snake_case_						)
						_UpperCAmelCase													=					param.transpose(0     ,   1						).contiguous()
						_UpperCAmelCase													=					param.view(*snake_case_						)
						return param


def 							__SCREAMING_SNAKE_CASE  (      snake_case_     ,   snake_case_						):

						'''simple docstring'''


						_UpperCAmelCase													=					config.n_positions
						_UpperCAmelCase													=					config.n_layer
						_UpperCAmelCase													=					config.vocab_size
						_UpperCAmelCase													=					config.n_head
						_UpperCAmelCase													=					config.n_embd // config.n_head

						_UpperCAmelCase													=					sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
						_UpperCAmelCase													=					{
						    "transformer.wte.weight": word_embeddings,
						    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
						    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
						    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
						}

						_UpperCAmelCase													=					"model.language_model.encoder.layers."
						for i in range(snake_case_						):
												_UpperCAmelCase													=					torch.tril(torch.ones((n_positions, n_positions)     ,   dtype=torch.bool						)						)
												_UpperCAmelCase													=					causal_mask.view(1     ,   1     ,   snake_case_     ,   snake_case_						)
												_UpperCAmelCase													=					causal_mask
												_UpperCAmelCase													=					torch.tensor(-1e4     ,   dtype=torch.bfloataa						)

												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
												_UpperCAmelCase													=					fix_query_key_value_ordering(snake_case_     ,   3     ,   snake_case_     ,   snake_case_						)
												_UpperCAmelCase													=					vala.transpose(0     ,   1						).contiguous()

												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
												_UpperCAmelCase													=					fix_query_key_value_ordering(snake_case_     ,   3     ,   snake_case_     ,   snake_case_						)
												_UpperCAmelCase													=					vala

												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
												    0     ,   1						)
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0     ,   1						)
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
												    0     ,   1						)
												_UpperCAmelCase													=					sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

						# For LM head, transformers' wants the matrix to weight embeddings.
						_UpperCAmelCase													=					word_embeddings

						return sd_hf


def 							__SCREAMING_SNAKE_CASE  (      snake_case_     ,   snake_case_						):

						'''simple docstring'''


						_UpperCAmelCase													=					6_4000
						_UpperCAmelCase													=					config_megatron["encoder_seq_length"]
						_UpperCAmelCase													=					config_megatron["hidden_size"]
						_UpperCAmelCase													=					config_megatron["num_layers"]
						_UpperCAmelCase													=					config_megatron["num_attention_heads"]
						_UpperCAmelCase													=					config_megatron["ffn_hidden_size"]
						_UpperCAmelCase													=					"gelu"
						_UpperCAmelCase													=					0.1
						_UpperCAmelCase													=					0.1
						_UpperCAmelCase													=					0.1
						_UpperCAmelCase													=					config_megatron["layernorm_epsilon"]  # 1e-5
						_UpperCAmelCase													=					config_megatron["init_method_std"]  # 0.02
						_UpperCAmelCase													=					config_megatron["apply_query_key_layer_scaling"]  # True
						_UpperCAmelCase													=					True
						_UpperCAmelCase													=					True

						# This identifies the 6.7B (7B) model which uses a different tokenizer
						if config_megatron["hidden_size"] == 4096:
												_UpperCAmelCase													=					1  # <|endoftext|>
												_UpperCAmelCase													=					1  # <|endoftext|>
												_UpperCAmelCase													=					0  # <unk>
						else:
												_UpperCAmelCase													=					2  # <s>
												_UpperCAmelCase													=					3  # <|endoftext|>
												_UpperCAmelCase													=					0  # <pad>

						return config_hf


def 							__SCREAMING_SNAKE_CASE  (      snake_case_						):

						'''simple docstring'''


						print(snake_case_						)

						_UpperCAmelCase													=					args.checkpoint_path
						_UpperCAmelCase													=					args.save_path
						if isfile(snake_case_						):
												raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""						)

						# Load the model.
						_UpperCAmelCase													=					torch.load(snake_case_     ,   map_location="cpu"						)

						# Load the config.
						_UpperCAmelCase													=					checkpoint["hyper_parameters"]["cfg"]
						_UpperCAmelCase													=					GPTaConfig()
						_UpperCAmelCase													=					copy_config(config_hf=snake_case_     ,   config_megatron=snake_case_						)
						_UpperCAmelCase													=					["GPT2LMHeadModel"]

						_UpperCAmelCase													=					checkpoint["state_dict"]

						# Convert.
						print("Converting"						)
						_UpperCAmelCase													=					convert_megatron_checkpoint(snake_case_     ,   snake_case_						)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(snake_case_     ,   snake_case_						)

						_UpperCAmelCase													=					"GPTSw3Tokenizer"

						# Store the config to file.
						print("Saving config"						)
						config_hf.save_pretrained(snake_case_						)

						# Store the state_dict to file.
						_UpperCAmelCase													=					os.path.join(snake_case_     ,   "pytorch_model.bin"						)
						print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""						)
						torch.save(snake_case_     ,   snake_case_						)


if __name__ == "__main__":
				lowercase_     :    Any      =			argparse.ArgumentParser()
				parser.add_argument(
				    '--checkpoint_path',
				    type=str,
				    required=True,
				    help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
				)
				parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
				parser.add_argument('--print-checkpoint-structure', action='store_true')
				lowercase_     :    Dict      =			parser.parse_args()
				main(_args)










from __future__ import annotations


def 							__SCREAMING_SNAKE_CASE  (      snake_case_						):

						'''simple docstring'''


						if len(snake_case_						) == 0:
												raise ValueError("find_max() arg is an empty sequence"						)
						_UpperCAmelCase													=					nums[0]
						for x in nums:
												if x > max_num:
																		_UpperCAmelCase													=					x
						return max_num


if __name__ == "__main__":
				import doctest

				doctest.testmod(verbose=True)




