






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig
def        a_	( _A ,				_A ,				_A=0					)     ->     Optional[Any]:







				"""simple docstring"""







				if name is None:
								snake_case__													=       None
				else:
								snake_case__													=       '.' * max(0 ,				spaces - 2					) + '# {:' + str(50 - spaces					) + 's}'
								snake_case__													=       fmt.format(__UpperCamelCase					)

				# Print and recurse (if needed).
				if isinstance(__UpperCamelCase ,				__UpperCamelCase					):
								if msg is not None:
												print(__UpperCamelCase					)
								for k in val.keys():
												recursive_print(__UpperCamelCase ,				val[k] ,				spaces + 2					)
				elif isinstance(__UpperCamelCase ,				torch.Tensor					):
								print(__UpperCamelCase ,				':' ,				val.size()					)
				else:
								print(__UpperCamelCase ,				':' ,				__UpperCamelCase					)
def        a_	( _A ,				_A ,				_A ,				_A					)     ->     Dict:







				"""simple docstring"""
				snake_case__													=       param.size()
				# other versions store [num_heads * num_splits * hidden_size, :]
				snake_case__													=       (num_heads, num_splits, hidden_size) + input_shape[1:]
				snake_case__													=       param.view(*__UpperCamelCase					)
				snake_case__													=       param.transpose(0 ,				1					).contiguous()
				snake_case__													=       param.view(*__UpperCamelCase					)
				return param
def        a_	( _A ,				_A					)     ->     Tuple:







				"""simple docstring"""
				snake_case__													=       config.n_positions
				snake_case__													=       config.n_layer
				snake_case__													=       config.vocab_size
				snake_case__													=       config.n_head
				snake_case__													=       config.n_embd // config.n_head

				snake_case__													=       sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
				snake_case__													=       {
				    'transformer.wte.weight': word_embeddings,
				    'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
				    'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
				    'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
				}

				snake_case__													=       'model.language_model.encoder.layers.'
				for i in range(__UpperCamelCase					):
								snake_case__													=       torch.tril(torch.ones((n_positions, n_positions) ,				dtype=torch.bool					)					)
								snake_case__													=       causal_mask.view(1 ,				1 ,				__UpperCamelCase ,				__UpperCamelCase					)
								snake_case__													=       causal_mask
								snake_case__													=       torch.tensor(-1e4 ,				dtype=torch.bfloataa					)

								snake_case__													=       sd_megatron[f'''{pf}{i}.input_layernorm.weight''']
								snake_case__													=       sd_megatron[f'''{pf}{i}.input_layernorm.bias''']

								snake_case__													=       sd_megatron[f'''{pf}{i}.self_attention.query_key_value.weight''']
								snake_case__													=       fix_query_key_value_ordering(__UpperCamelCase ,				3 ,				__UpperCamelCase ,				__UpperCamelCase					)
								snake_case__													=       vala.transpose(0 ,				1					).contiguous()

								snake_case__													=       sd_megatron[f'''{pf}{i}.self_attention.query_key_value.bias''']
								snake_case__													=       fix_query_key_value_ordering(__UpperCamelCase ,				3 ,				__UpperCamelCase ,				__UpperCamelCase					)
								snake_case__													=       vala

								snake_case__													=       sd_megatron[f'''{pf}{i}.self_attention.dense.weight'''].transpose(
								    0 ,				1					)
								snake_case__													=       sd_megatron[f'''{pf}{i}.self_attention.dense.bias''']
								snake_case__													=       sd_megatron[f'''{pf}{i}.post_attention_layernorm.weight''']
								snake_case__													=       sd_megatron[f'''{pf}{i}.post_attention_layernorm.bias''']
								snake_case__													=       sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0 ,				1					)
								snake_case__													=       sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.bias''']
								snake_case__													=       sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
								    0 ,				1					)
								snake_case__													=       sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.bias''']

				# For LM head, transformers' wants the matrix to weight embeddings.
				snake_case__													=       word_embeddings

				return sd_hf
def        a_	( _A ,				_A					)     ->     Optional[int]:







				"""simple docstring"""
				snake_case__													=       64000
				snake_case__													=       config_megatron['encoder_seq_length']
				snake_case__													=       config_megatron['hidden_size']
				snake_case__													=       config_megatron['num_layers']
				snake_case__													=       config_megatron['num_attention_heads']
				snake_case__													=       config_megatron['ffn_hidden_size']
				snake_case__													=       'gelu'
				snake_case__													=       0.1
				snake_case__													=       0.1
				snake_case__													=       0.1
				snake_case__													=       config_megatron['layernorm_epsilon']  # 1e-5
				snake_case__													=       config_megatron['init_method_std']  # 0.02
				snake_case__													=       config_megatron['apply_query_key_layer_scaling']  # True
				snake_case__													=       True
				snake_case__													=       True

				# This identifies the 6.7B (7B) model which uses a different tokenizer
				if config_megatron["hidden_size"] == 4096:
								snake_case__													=       1  # <|endoftext|>
								snake_case__													=       1  # <|endoftext|>
								snake_case__													=       0  # <unk>
				else:
								snake_case__													=       2  # <s>
								snake_case__													=       3  # <|endoftext|>
								snake_case__													=       0  # <pad>

				return config_hf
def        a_	( _A					)     ->     str:







				"""simple docstring"""
				print(__UpperCamelCase					)

				snake_case__													=       args.checkpoint_path
				snake_case__													=       args.save_path
				if isfile(__UpperCamelCase					):
								raise FileNotFoundError(f'''ERROR! could not find file {checkpoint_path}'''					)

				# Load the model.
				snake_case__													=       torch.load(__UpperCamelCase ,				map_location='cpu'					)

				# Load the config.
				snake_case__													=       checkpoint['hyper_parameters']['cfg']
				snake_case__													=       GPTaConfig()
				snake_case__													=       copy_config(config_hf=__UpperCamelCase ,				config_megatron=__UpperCamelCase					)
				snake_case__													=       ['GPT2LMHeadModel']

				snake_case__													=       checkpoint['state_dict']

				# Convert.
				print('Converting'					)
				snake_case__													=       convert_megatron_checkpoint(__UpperCamelCase ,				__UpperCamelCase					)

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(__UpperCamelCase ,				__UpperCamelCase					)

				snake_case__													=       'GPTSw3Tokenizer'

				# Store the config to file.
				print('Saving config'					)
				config_hf.save_pretrained(__UpperCamelCase					)

				# Store the state_dict to file.
				snake_case__													=       os.path.join(__UpperCamelCase ,				'pytorch_model.bin'					)
				print(f'''Saving checkpoint to "{output_checkpoint_file}"'''					)
				torch.save(__UpperCamelCase ,				__UpperCamelCase					)


if __name__ == "__main__":
	__UpperCamelCase   :						Any							=  argparse.ArgumentParser()
	parser.add_argument(
	    """--checkpoint_path""",
	    type=str,
	    required=True,
	    help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
	)
	parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
	parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
	__UpperCamelCase   :						Union[str, Any]							=  parser.parse_args()
	main(_args)












import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
__UpperCamelCase   :						List[Any]							=  logging.getLogger(__name__)
def        a_	( )     ->     int:







				"""simple docstring"""
				snake_case__													=       argparse.ArgumentParser(description='PyTorch Transformer Language Model'					)
				parser.add_argument('--model_name' ,				type=_A ,				default='transfo-xl-wt103' ,				help='pretrained model name'					)
				parser.add_argument(
				    '--split' ,				type=_A ,				default='test' ,				choices=['all', 'valid', 'test'] ,				help='which split to evaluate'					)
				parser.add_argument('--batch_size' ,				type=_A ,				default=10 ,				help='batch size'					)
				parser.add_argument('--tgt_len' ,				type=_A ,				default=128 ,				help='number of tokens to predict'					)
				parser.add_argument('--ext_len' ,				type=_A ,				default=0 ,				help='length of the extended context'					)
				parser.add_argument('--mem_len' ,				type=_A ,				default=1600 ,				help='length of the retained previous heads'					)
				parser.add_argument('--clamp_len' ,				type=_A ,				default=1000 ,				help='max positional embedding index'					)
				parser.add_argument('--no_cuda' ,				action='store_true' ,				help='Do not use CUDA even though CUA is available'					)
				parser.add_argument('--work_dir' ,				type=_A ,				required=_A ,				help='path to the work_dir'					)
				parser.add_argument('--no_log' ,				action='store_true' ,				help='do not log the eval result'					)
				parser.add_argument('--same_length' ,				action='store_true' ,				help='set same length attention with masking'					)
				parser.add_argument('--server_ip' ,				type=_A ,				default='' ,				help='Can be used for distant debugging.'					)
				parser.add_argument('--server_port' ,				type=_A ,				default='' ,				help='Can be used for distant debugging.'					)
				snake_case__													=       parser.parse_args()
				assert args.ext_len >= 0, "extended context length must be non-negative"

				if args.server_ip and args.server_port:
								# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
								import ptvsd

								print('Waiting for debugger attach'					)
								ptvsd.enable_attach(address=(args.server_ip, args.server_port) ,				redirect_output=_A					)
								ptvsd.wait_for_attach()

				snake_case__													=       torch.device('cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu'					)
				logger.info('device: {}'.format(_A					)					)

				# Load a pre-processed dataset
				# You can also build the corpus yourself using TransfoXLCorpus methods
				# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
				# and tokenizing the dataset
				# The pre-processed corpus is a convertion (using the conversion script )
				snake_case__													=       TransfoXLCorpus.from_pretrained(args.model_name					)

				snake_case__													=       corpus.get_iterator('valid' ,				args.batch_size ,				args.tgt_len ,				device=_A ,				ext_len=args.ext_len					)
				snake_case__													=       corpus.get_iterator('test' ,				args.batch_size ,				args.tgt_len ,				device=_A ,				ext_len=args.ext_len					)

				# Load a pre-trained model
				snake_case__													=       TransfoXLLMHeadModel.from_pretrained(args.model_name					)
				model.to(_A					)

				logger.info(
				    'Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format(
				        args.batch_size ,				args.tgt_len ,				args.ext_len ,				args.mem_len ,				args.clamp_len					)					)

				model.reset_memory_length(args.mem_len					)
				if args.clamp_len > 0:
								snake_case__													=       args.clamp_len
				if args.same_length:
								snake_case__													=       True

				###############################################################################
				# Evaluation code
				###############################################################################
				def evaluate(_A					):
								# Turn on evaluation mode which disables dropout.
								model.eval()
								snake_case__							,  snake_case__													=       0, 0.0
								snake_case__													=       time.time()
								with torch.no_grad():
												snake_case__													=       None
												for idx, (data, target, seq_len) in enumerate(_A					):
																snake_case__													=       model(_A ,				lm_labels=_A ,				mems=_A					)
																snake_case__							,  snake_case__							,  snake_case__													=       ret
																snake_case__													=       loss.mean()
																total_loss += seq_len * loss.item()
																total_len += seq_len
												snake_case__													=       time.time() - start_time
								logger.info('Time : {:.2f}s, {:.2f}ms/segment'.format(_A ,				1000 * total_time / (idx + 1)					)					)
								return total_loss / total_len

				# Run on test data.
				if args.split == "all":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       evaluate(_A					)
				elif args.split == "valid":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       None
				elif args.split == "test":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       None

				def format_log(_A ,				_A					):
								snake_case__													=       '| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format(_A ,				_A ,				math.exp(_A					)					)
								return log_str

				snake_case__													=       ''
				if valid_loss is not None:
								log_str += format_log(_A ,				'valid'					)
				if test_loss is not None:
								log_str += format_log(_A ,				'test'					)

				logger.info('=' * 100					)
				logger.info(_A					)
				logger.info('=' * 100					)


if __name__ == "__main__":
							main()




