



# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		List[Any]			,    SCREAMING_SNAKE_CASE__    :		Optional[int]			,    SCREAMING_SNAKE_CASE__    :		Any=0			)					->       Any:
 # Format the message.
 if name is None:
  _snake_case      :					Tuple					=   None
 else:
  _snake_case      :					Union[str, Any]					=   """.""" * max(0			,    spaces - 2			) + """# {:""" + str(50 - spaces			) + """s}"""
  _snake_case      :					str					=   fmt.format(SCREAMING_SNAKE_CASE__			)

 # Print and recurse (if needed).
 if isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  if msg is not None:
   print(SCREAMING_SNAKE_CASE__			)
  for k in val.keys():
   recursive_print(SCREAMING_SNAKE_CASE__			,    val[k]			,    spaces + 2			)
 elif isinstance(SCREAMING_SNAKE_CASE__			,    torch.Tensor			):
  print(SCREAMING_SNAKE_CASE__			,    """:"""			,    val.size()			)
 else:
  print(SCREAMING_SNAKE_CASE__			,    """:"""			,    SCREAMING_SNAKE_CASE__			)



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Tuple			,    SCREAMING_SNAKE_CASE__    :		List[str]			,    SCREAMING_SNAKE_CASE__    :		Tuple			,    SCREAMING_SNAKE_CASE__    :		Optional[int]			,    SCREAMING_SNAKE_CASE__    :		Tuple			)					->       Union[str, Any]:
 # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
 # for compatibility with later versions of NVIDIA Megatron-LM.
 # The inverse operation is performed inside Megatron-LM to read checkpoints:
 # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
 # If param is the weight tensor of the self-attention block, the returned tensor
 # will have to be transposed one more time to be read by HuggingFace BERT.
 _snake_case      :					Dict					=   param.size()
 if checkpoint_version == 1.0:
  # version 1.0 stores [num_heads * hidden_size * num_splits, :]
  _snake_case      :					List[Any]					=   (num_heads, hidden_size, num_splits) + input_shape[1:]
  _snake_case      :					List[str]					=   param.view(*SCREAMING_SNAKE_CASE__			)
  _snake_case      :					str					=   param.transpose(0			,    2			)
  _snake_case      :					Tuple					=   param.transpose(1			,    2			).contiguous()
 elif checkpoint_version >= 2.0:
  # other versions store [num_heads * num_splits * hidden_size, :]
  _snake_case      :					int					=   (num_heads, num_splits, hidden_size) + input_shape[1:]
  _snake_case      :					Union[str, Any]					=   param.view(*SCREAMING_SNAKE_CASE__			)
  _snake_case      :					Union[str, Any]					=   param.transpose(0			,    1			).contiguous()
 _snake_case      :					str					=   param.view(*SCREAMING_SNAKE_CASE__			)
 return param



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Tuple			,    SCREAMING_SNAKE_CASE__    :		str			,    SCREAMING_SNAKE_CASE__    :		int			)					->       str:
 # The converted output model.
 _snake_case      :					Union[str, Any]					=   {}

 # old versions did not store training args
 _snake_case      :					Optional[int]					=   input_state_dict.get("""args"""			,    SCREAMING_SNAKE_CASE__			)
 if ds_args is not None:
  # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
  # from pprint import pprint
  # pprint(vars(ds_args))

  _snake_case      :					int					=   ds_args.tokenizer_type
  _snake_case      :					int					=   ds_args.padded_vocab_size
  _snake_case      :					Optional[Any]					=   ds_args.max_position_embeddings
  _snake_case      :					List[Any]					=   ds_args.hidden_size
  _snake_case      :					Dict					=   ds_args.num_layers
  _snake_case      :					Optional[Any]					=   ds_args.num_attention_heads
  _snake_case      :					Optional[int]					=   ds_args.ffn_hidden_size if """ffn_hidden_size""" in ds_args else 4 * ds_args.hidden_size
  # pprint(config)

 # The number of heads.
 _snake_case      :					str					=   config.num_attention_heads
 # The hidden_size per head.
 _snake_case      :					Optional[Any]					=   config.hidden_size // heads
 # Megatron-LM checkpoint version
 if "checkpoint_version" in input_state_dict.keys():
  _snake_case      :					Union[str, Any]					=   input_state_dict["""checkpoint_version"""]
 else:
  _snake_case      :					int					=   0.0

 # The model.
 _snake_case      :					Optional[Any]					=   input_state_dict["""model"""]
 # The language model.
 _snake_case      :					str					=   model["""language_model"""]
 # The embeddings.
 _snake_case      :					Optional[Any]					=   lm["""embedding"""]

 # The word embeddings.
 _snake_case      :					Union[str, Any]					=   embeddings["""word_embeddings"""]["""weight"""]
 # Truncate the embedding table to vocab_size rows.
 _snake_case      :					List[str]					=   word_embeddings[: config.vocab_size, :]
 # Store the word embeddings.
 _snake_case      :					Optional[int]					=   word_embeddings

 # The position embeddings.
 _snake_case      :					str					=   embeddings["""position_embeddings"""]["""weight"""]
 assert pos_embeddings.size(0			) == config.max_position_embeddings and pos_embeddings.size(1			) == config.hidden_size
 # Store the position embeddings.
 _snake_case      :					Union[str, Any]					=   pos_embeddings

 # The token-type embeddings.
 _snake_case      :					Tuple					=   embeddings["""tokentype_embeddings"""]["""weight"""]
 # Store the position embeddings.
 _snake_case      :					int					=   tokentype_embeddings

 # The transformer.
 _snake_case      :					Tuple					=   lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

 # The regex to extract layer names.
 _snake_case      :					Dict					=   re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""			)

 # The simple map of names for "automated" rules.
 _snake_case      :					Tuple					=   {
     """attention.dense""": """.attention.output.dense.""",
     """self_attention.dense""": """.attention.output.dense.""",
     """mlp.dense_h_to_4h""": """.intermediate.dense.""",
     """mlp.dense_4h_to_h""": """.output.dense.""",
 }

 # Keep track of the attention/query/value tensor.
 _snake_case      :					Optional[Any]					=   None

 # Extract the layers.
 for key, val in transformer.items():
  # Match the name.
  _snake_case      :					List[Any]					=   layer_re.match(SCREAMING_SNAKE_CASE__			)

  # Stop if that's not a layer
  if m is None:
   break

  # The index of the layer.
  _snake_case      :					int					=   int(m.group(1			)			)
  # The name of the operation.
  _snake_case      :					Tuple					=   m.group(2			)
  # Is it a weight or a bias?
  _snake_case      :					Dict					=   m.group(3			)

  # The name of the layer.
  _snake_case      :					Optional[int]					=   F'''bert.encoder.layer.{layer_idx}'''

  # For layernorm(s), simply store the layer norm.
  if op_name.endswith("""layernorm"""			):
   _snake_case      :					Union[str, Any]					=   """attention.ln""" if op_name.startswith("""input"""			) else """ln"""
   _snake_case      :					Tuple					=   val

  # Transpose the QKV matrix.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "weight":
   # Make sure the QKV pointer is nil.
   assert attention_qkv_weight is None, ""

   _snake_case      :					Any					=   fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			,    3			,    SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			)
   # Store the tensor as we need the bias as well to interleave QKV and biases.
   _snake_case      :					List[Any]					=   out_val

  # Transpose the bias.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "bias":
   # Make sure we read the weight tensor.
   assert attention_qkv_weight is not None, ""

   # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
   _snake_case      :					Union[str, Any]					=   attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
   _snake_case      :					Optional[Any]					=   attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
   _snake_case      :					Optional[Any]					=   attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

   _snake_case      :					Optional[Any]					=   fix_query_key_value_ordering(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			,    3			,    SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			)
   # Split the bias.
   _snake_case      :					Tuple					=   out_val[0 * config.hidden_size : 1 * config.hidden_size]
   _snake_case      :					List[Any]					=   out_val[1 * config.hidden_size : 2 * config.hidden_size]
   _snake_case      :					Union[str, Any]					=   out_val[2 * config.hidden_size : 3 * config.hidden_size]

   # Store.
   _snake_case      :					str					=   q
   _snake_case      :					List[Any]					=   q_bias
   _snake_case      :					int					=   k
   _snake_case      :					Union[str, Any]					=   k_bias
   _snake_case      :					List[str]					=   v
   _snake_case      :					Tuple					=   v_bias

   # Clear the stored tensor.
   _snake_case      :					List[str]					=   None

  # Copy weights and biases as is.
  elif weight_or_bias in ["weight", "bias"]:
   _snake_case      :					List[Any]					=   megatron_to_transformers[op_name]
   _snake_case      :					List[str]					=   val

    # The final layernorm.
 _snake_case      :					Optional[int]					=   transformer["""final_layernorm.weight"""]
 _snake_case      :					Optional[int]					=   transformer["""final_layernorm.bias"""]

 # The pooler.
 _snake_case      :					Optional[Any]					=   lm["""pooler"""]

 # Store the matrix and the bias.
 _snake_case      :					Any					=   pooler["""dense.weight"""]
 _snake_case      :					Union[str, Any]					=   pooler["""dense.bias"""]

 # The LM head from Megatron (for RACE).
 _snake_case      :					Optional[Any]					=   model["""lm_head"""]

 # The transform matrix.
 _snake_case      :					str					=   lm_head["""dense.weight"""]
 _snake_case      :					Union[str, Any]					=   lm_head["""dense.bias"""]

 # The transform LN.
 _snake_case      :					Optional[int]					=   lm_head["""layernorm.weight"""]
 _snake_case      :					Union[str, Any]					=   lm_head["""layernorm.bias"""]

 # For the decoder, we replicate the weights.
 _snake_case      :					Optional[int]					=   word_embeddings
 _snake_case      :					Any					=   lm_head["""bias"""]

 # The classifier from Megatron (for MLNI).
 _snake_case      :					Dict					=   model["""binary_head"""]

 # Store the classifier.
 _snake_case      :					Union[str, Any]					=   binary_head["""weight"""]
 _snake_case      :					int					=   binary_head["""bias"""]

 # It should be done!
 return output_state_dict



def  lowercase    (				)					->       Optional[Any]:
 # Create the argument parser.
 _snake_case      :					List[str]					=   argparse.ArgumentParser()
 parser.add_argument("""--print-checkpoint-structure"""			,    action="""store_true"""			)
 parser.add_argument("""path_to_checkpoint"""			,    type=SCREAMING_SNAKE_CASE__			,    help="""Path to the ZIP file containing the checkpoint"""			)
 parser.add_argument(
     """--config_file"""			,    default=""""""			,    type=SCREAMING_SNAKE_CASE__			,    help="""An optional config json file describing the pre-trained model."""			,    )
 _snake_case      :					Tuple					=   parser.parse_args()

 # Extract the basename.
 _snake_case      :					Tuple					=   os.path.dirname(args.path_to_checkpoint			)

 # Load the model.
 # the .zip is very optional, let's keep it for backward compatibility
 print(F'''Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'''			)
 if args.path_to_checkpoint.endswith(""".zip"""			):
  with zipfile.ZipFile(args.path_to_checkpoint			,    """r"""			) as checkpoint:
   with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""			) as pytorch_dict:
    _snake_case      :					Optional[Any]					=   torch.load(SCREAMING_SNAKE_CASE__			,    map_location="""cpu"""			)
 else:
  _snake_case      :					Optional[int]					=   torch.load(args.path_to_checkpoint			,    map_location="""cpu"""			)

 if args.config_file == "":
  # Default config of megatron-bert 345m
  _snake_case      :					Union[str, Any]					=   MegatronBertConfig()

  # different megatron-bert-*-345m models have different vocab sizes, so override the default
  # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
  _snake_case      :					Optional[int]					=   input_state_dict["""model"""]["""lm_head"""]["""bias"""].numel()
 else:
  _snake_case      :					Optional[int]					=   MegatronBertConfig.from_json_file(args.config_file			)

 # Convert.
 print("""Converting"""			)
 _snake_case      :					Optional[int]					=   convert_megatron_checkpoint(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			)

 # Print the structure of converted state dict.
 if args.print_checkpoint_structure:
  recursive_print(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			)

 # Store the config to file.
 print("""Saving config"""			)
 config.save_pretrained(SCREAMING_SNAKE_CASE__			)

 # Store the state_dict to file.
 _snake_case      :					Any					=   os.path.join(SCREAMING_SNAKE_CASE__			,    """pytorch_model.bin"""			)
 print(F'''Saving checkpoint to "{output_checkpoint_file}"'''			)
 torch.save(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			)


####################################################################################################

if __name__ == "__main__":
  main()

####################################################################################################





def  lowercase    (				SCREAMING_SNAKE_CASE__    :		int			)					->       int:
 if not isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  raise TypeError("""recursive_lucas_number accepts only integer arguments."""			)
 if n_th_number == 0:
  return 2
 if n_th_number == 1:
  return 1

 return recursive_lucas_number(n_th_number - 1			) + recursive_lucas_number(
     n_th_number - 2			)



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		int			)					->       int:
 if not isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  raise TypeError("""dynamic_lucas_number accepts only integer arguments."""			)
 _snake_case		,      _snake_case      :					Any					=   2, 1
 for _ in range(SCREAMING_SNAKE_CASE__			):
  _snake_case		,      _snake_case      :					Union[str, Any]					=   b, a + b
 return a


if __name__ == "__main__":
  from doctest import testmod

  testmod()
  a__      =		int(input("""Enter the number of terms in lucas series:\n""").strip())
  print("""Using recursive function to calculate lucas series:""")
  print(""" """.join(str(recursive_lucas_number(i)) for i in range(n)))
  print("""\nUsing dynamic function to calculate lucas series:""")
  print(""" """.join(str(dynamic_lucas_number(i)) for i in range(n)))
