
import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def       UpperCAmelCase_					(  __lowerCAmelCase		,   __lowerCAmelCase		,   __lowerCAmelCase=0		)	->  Union[str, Any]:
    # Format the message.
    if name is None:
        __lowercase						:     Union[str, Any]		      =    None
    else:
        __lowercase						:     Optional[int]		      =    '''.''' * max(0		,   spaces - 2		) + '''# {:''' + str(50 - spaces		) + '''s}'''
        __lowercase						:     Dict		      =    fmt.format(__lowerCAmelCase		)

    # Print and recurse (if needed).
    if isinstance(__lowerCAmelCase		,   __lowerCAmelCase		):
        if msg is not None:
            print(__lowerCAmelCase		)
        for k in val.keys():
            recursive_print(__lowerCAmelCase		,   val[k]		,   spaces + 2		)
    elif isinstance(__lowerCAmelCase		,   torch.Tensor		):
        print(__lowerCAmelCase		,   ''':'''		,   val.size()		)
    else:
        print(__lowerCAmelCase		,   ''':'''		,   __lowerCAmelCase		)


def       UpperCAmelCase_					(  __lowerCAmelCase		,   __lowerCAmelCase		,   __lowerCAmelCase		,   __lowerCAmelCase		)	->  List[str]:
    # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
    # for compatibility with later versions of NVIDIA Megatron-LM.
    # The inverse operation is performed inside Megatron-LM to read checkpoints:
    # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
    # If param is the weight tensor of the self-attention block, the returned tensor
    # will have to be transposed one more time to be read by HuggingFace GPT2.
    __lowercase						:     Any		      =    param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    __lowercase						:     str		      =    (num_heads, num_splits, hidden_size) + input_shape[1:]
    __lowercase						:     Optional[int]		      =    param.view(*__lowerCAmelCase		)
    __lowercase						:     Union[str, Any]		      =    param.transpose(0		,   1		).contiguous()
    __lowercase						:     Union[str, Any]		      =    param.view(*__lowerCAmelCase		)
    return param


def       UpperCAmelCase_					(  __lowerCAmelCase		,   __lowerCAmelCase		)	->  int:
    __lowercase						:     str		      =    config.n_positions
    __lowercase						:     int		      =    config.n_layer
    __lowercase						:     Dict		      =    config.vocab_size
    __lowercase						:     Optional[int]		      =    config.n_head
    __lowercase						:     Dict		      =    config.n_embd // config.n_head

    __lowercase						:     Dict		      =    sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
    __lowercase						:     List[Any]		      =    {
        '''transformer.wte.weight''': word_embeddings,
        '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
        '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
        '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
    }

    __lowercase						:     List[str]		      =    '''model.language_model.encoder.layers.'''
    for i in range(__lowerCAmelCase		):
        __lowercase						:     Any		      =    torch.tril(torch.ones((n_positions, n_positions)		,   dtype=torch.bool		)		)
        __lowercase						:     Optional[int]		      =    causal_mask.view(1		,   1		,   __lowerCAmelCase		,   __lowerCAmelCase		)
        __lowercase						:     List[Any]		      =    causal_mask
        __lowercase						:     Tuple		      =    torch.tensor(-1E4		,   dtype=torch.bfloataa		)

        __lowercase						:     List[str]		      =    sd_megatron[F'{pf}{i}.input_layernorm.weight']
        __lowercase						:     int		      =    sd_megatron[F'{pf}{i}.input_layernorm.bias']

        __lowercase						:     Optional[int]		      =    sd_megatron[F'{pf}{i}.self_attention.query_key_value.weight']
        __lowercase						:     List[str]		      =    fix_query_key_value_ordering(__lowerCAmelCase		,   3		,   __lowerCAmelCase		,   __lowerCAmelCase		)
        __lowercase						:     Tuple		      =    vala.transpose(0		,   1		).contiguous()

        __lowercase						:     int		      =    sd_megatron[F'{pf}{i}.self_attention.query_key_value.bias']
        __lowercase						:     List[Any]		      =    fix_query_key_value_ordering(__lowerCAmelCase		,   3		,   __lowerCAmelCase		,   __lowerCAmelCase		)
        __lowercase						:     Optional[int]		      =    vala

        __lowercase						:     List[str]		      =    sd_megatron[F'{pf}{i}.self_attention.dense.weight'].transpose(
            0		,   1		)
        __lowercase						:     Dict		      =    sd_megatron[F'{pf}{i}.self_attention.dense.bias']
        __lowercase						:     Optional[Any]		      =    sd_megatron[F'{pf}{i}.post_attention_layernorm.weight']
        __lowercase						:     str		      =    sd_megatron[F'{pf}{i}.post_attention_layernorm.bias']
        __lowercase						:     str		      =    sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0		,   1		)
        __lowercase						:     Any		      =    sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.bias']
        __lowercase						:     int		      =    sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
            0		,   1		)
        __lowercase						:     int		      =    sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.bias']

    # For LM head, transformers' wants the matrix to weight embeddings.
    __lowercase						:     str		      =    word_embeddings

    return sd_hf


def       UpperCAmelCase_					(  __lowerCAmelCase		,   __lowerCAmelCase		)	->  Union[str, Any]:
    __lowercase						:     Optional[int]		      =    64_000
    __lowercase						:     List[str]		      =    config_megatron['''encoder_seq_length''']
    __lowercase						:     Optional[Any]		      =    config_megatron['''hidden_size''']
    __lowercase						:     Union[str, Any]		      =    config_megatron['''num_layers''']
    __lowercase						:     Optional[Any]		      =    config_megatron['''num_attention_heads''']
    __lowercase						:     Dict		      =    config_megatron['''ffn_hidden_size''']
    __lowercase						:     int		      =    '''gelu'''
    __lowercase						:     List[str]		      =    0.1
    __lowercase						:     str		      =    0.1
    __lowercase						:     Optional[int]		      =    0.1
    __lowercase						:     Optional[int]		      =    config_megatron['''layernorm_epsilon''']  # 1e-5
    __lowercase						:     Optional[int]		      =    config_megatron['''init_method_std''']  # 0.02
    __lowercase						:     str		      =    config_megatron['''apply_query_key_layer_scaling''']  # True
    __lowercase						:     Optional[Any]		      =    True
    __lowercase						:     int		      =    True

    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4_096:
        __lowercase						:     Optional[Any]		      =    1  # <|endoftext|>
        __lowercase						:     Optional[int]		      =    1  # <|endoftext|>
        __lowercase						:     Union[str, Any]		      =    0  # <unk>
    else:
        __lowercase						:     List[Any]		      =    2  # <s>
        __lowercase						:     Optional[int]		      =    3  # <|endoftext|>
        __lowercase						:     List[str]		      =    0  # <pad>

    return config_hf


def       UpperCAmelCase_					(  __lowerCAmelCase		)	->  Any:
    print(__lowerCAmelCase		)

    __lowercase						:     List[str]		      =    args.checkpoint_path
    __lowercase						:     Optional[Any]		      =    args.save_path
    if isfile(__lowerCAmelCase		):
        raise FileNotFoundError(F'ERROR! could not find file {checkpoint_path}'		)

    # Load the model.
    __lowercase						:     str		      =    torch.load(__lowerCAmelCase		,   map_location='''cpu'''		)

    # Load the config.
    __lowercase						:     Optional[int]		      =    checkpoint['''hyper_parameters''']['''cfg''']
    __lowercase						:     Tuple		      =    GPTaConfig()
    __lowercase						:     str		      =    copy_config(config_hf=__lowerCAmelCase		,   config_megatron=__lowerCAmelCase		)
    __lowercase						:     Tuple		      =    ['''GPT2LMHeadModel''']

    __lowercase						:     int		      =    checkpoint['''state_dict''']

    # Convert.
    print('''Converting'''		)
    __lowercase						:     Any		      =    convert_megatron_checkpoint(__lowerCAmelCase		,   __lowerCAmelCase		)

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(__lowerCAmelCase		,   __lowerCAmelCase		)

    __lowercase						:     List[str]		      =    '''GPTSw3Tokenizer'''

    # Store the config to file.
    print('''Saving config'''		)
    config_hf.save_pretrained(__lowerCAmelCase		)

    # Store the state_dict to file.
    __lowercase						:     Optional[int]		      =    os.path.join(__lowerCAmelCase		,   '''pytorch_model.bin'''		)
    print(F'Saving checkpoint to "{output_checkpoint_file}"'		)
    torch.save(__lowerCAmelCase		,   __lowerCAmelCase		)


if __name__ == "__main__":
 __lowerCAmelCase   :     Any      =						argparse.ArgumentParser()
 parser.add_argument(
     "--checkpoint_path",
     type=str,
     required=True,
     help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
 )
 parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
 parser.add_argument("--print-checkpoint-structure", action="store_true")
 __lowerCAmelCase   :     List[Any]      =						parser.parse_args()
 main(_args)




import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer


class   __lowerCAmelCase		(		tf.keras.layers.Layer ):



      """simple docstring"""







      def __init__( self						:    int			,    _snake_case						:    List			,    _snake_case						:    bool			,    _snake_case						:    int = None			,    _snake_case						:    int = None			,    _snake_case						:    int = None			,    _snake_case						:    str = "longest"			,    _snake_case						:    bool = True			,    _snake_case						:    int = 512			,    _snake_case						:    int = None			,    _snake_case						:    bool = True			,    _snake_case						:    bool = True			,    _snake_case						:    bool = True			,    **_snake_case						:    int			,    ):
          super().__init__()
          if use_fast_bert_tokenizer:
              __lowercase						:     Optional[int]		      =    FastBertTokenizer(
                  _snake_case			,    token_out_type=tf.intaa			,    lower_case_nfd_strip_accents=_snake_case			,    **_snake_case			)
          else:
              __lowercase						:     List[str]		      =    tf.lookup.StaticVocabularyTable(
                  tf.lookup.KeyValueTensorInitializer(
                      keys=_snake_case			,    key_dtype=tf.string			,    values=tf.range(tf.size(_snake_case			,    out_type=tf.intaa			)			,    dtype=tf.intaa			)			,    value_dtype=tf.intaa			,    )			,    num_oov_buckets=1			,    )
              __lowercase						:     List[Any]		      =    BertTokenizerLayer(
                  _snake_case			,    token_out_type=tf.intaa			,    lower_case=_snake_case			,    **_snake_case			)

          __lowercase						:     Dict		      =    vocab_list
          __lowercase						:     str		      =    do_lower_case
          __lowercase						:     Optional[int]		      =    cls_token_id or vocab_list.index('''[CLS]'''			)
          __lowercase						:     Dict		      =    sep_token_id or vocab_list.index('''[SEP]'''			)
          __lowercase						:     List[str]		      =    pad_token_id or vocab_list.index('''[PAD]'''			)
          __lowercase						:     str		      =    ShrinkLongestTrimmer(max_length - 3			,    axis=1			)  # Allow room for special tokens
          __lowercase						:     Optional[Any]		      =    max_length
          __lowercase						:     Tuple		      =    padding
          __lowercase						:     str		      =    truncation
          __lowercase						:     str		      =    pad_to_multiple_of
          __lowercase						:     int		      =    return_token_type_ids
          __lowercase						:     List[str]		      =    return_attention_mask




      @classmethod
      def 						snake_case_     ( cls						:    Optional[Any]			,    _snake_case						:    "PreTrainedTokenizerBase"			,    **_snake_case						:    Dict			):  # noqa: F821
          __lowercase						:     Tuple		      =    kwargs.pop('''do_lower_case'''			,    _snake_case			)
          __lowercase						:     Union[str, Any]		      =    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
          __lowercase						:     Optional[Any]		      =    kwargs.pop('''cls_token_id'''			,    _snake_case			)
          __lowercase						:     str		      =    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
          __lowercase						:     Tuple		      =    kwargs.pop('''sep_token_id'''			,    _snake_case			)
          __lowercase						:     Any		      =    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
          __lowercase						:     Dict		      =    kwargs.pop('''pad_token_id'''			,    _snake_case			)
          __lowercase						:     List[Any]		      =    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

          __lowercase						:     Optional[int]		      =    tokenizer.get_vocab()
          __lowercase						:     Tuple		      =    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,    key=lambda _snake_case			: x[1]			)
          __lowercase						:     Optional[int]		      =    [entry[0] for entry in vocab]
          return cls(
              vocab_list=_snake_case			,    do_lower_case=_snake_case			,    cls_token_id=_snake_case			,    sep_token_id=_snake_case			,    pad_token_id=_snake_case			,    **_snake_case			,    )




      @classmethod
      def 						snake_case_     ( cls						:    Any			,    _snake_case						:    Union[str, os.PathLike]			,    *_snake_case						:    List[str]			,    **_snake_case						:    Tuple			):
          try:
              __lowercase						:     List[Any]		      =    BertTokenizer.from_pretrained(_snake_case			,    *_snake_case			,    **_snake_case			)
          except:  # noqa: E722
              from .tokenization_bert_fast import BertTokenizerFast

              __lowercase						:     List[str]		      =    BertTokenizerFast.from_pretrained(_snake_case			,    *_snake_case			,    **_snake_case			)
          return cls.from_tokenizer(_snake_case			,    **_snake_case			)




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    Union[str, Any]			):
          if self.do_lower_case:
              __lowercase						:     int		      =    case_fold_utfa(_snake_case			)
          __lowercase						:     Optional[int]		      =    self.tf_tokenizer.tokenize(_snake_case			)
          return tokens.merge_dims(1			,    -1			)




      def 						snake_case_     ( self						:    Tuple			,    _snake_case						:    Optional[int]			,    _snake_case						:    Any=None			,    _snake_case						:    List[Any]=None			,    _snake_case						:    Any=None			,    _snake_case						:    Dict=None			,    _snake_case						:    List[Any]=None			,    _snake_case						:    str=None			,    _snake_case						:    List[Any]=None			,    ):
          if padding is None:
              __lowercase						:     List[Any]		      =    self.padding
          if padding not in ("longest", "max_length"):
              raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''			)
          if max_length is not None and text_pair is not None:
              # Because we have to instantiate a Trimmer to do it properly
              raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''			)
          if max_length is None:
              __lowercase						:     Dict		      =    self.max_length
          if truncation is None:
              __lowercase						:     int		      =    self.truncation
          if pad_to_multiple_of is None:
              __lowercase						:     Tuple		      =    self.pad_to_multiple_of
          if return_token_type_ids is None:
              __lowercase						:     int		      =    self.return_token_type_ids
          if return_attention_mask is None:
              __lowercase						:     Optional[int]		      =    self.return_attention_mask
          if not isinstance(_snake_case			,    tf.Tensor			):
              __lowercase						:     Dict		      =    tf.convert_to_tensor(_snake_case			)
          if text_pair is not None and not isinstance(_snake_case			,    tf.Tensor			):
              __lowercase						:     Optional[Any]		      =    tf.convert_to_tensor(_snake_case			)
          if text_pair is not None:
              if text.shape.rank > 1:
                  raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''			)
              if text_pair.shape.rank > 1:
                  raise ValueError('''text_pair should not be multidimensional!'''			)
          if text.shape.rank == 2:
              __lowercase      , __lowercase						:     int		      =    text[:, 0], text[:, 1]
          __lowercase						:     int		      =    self.unpaired_tokenize(_snake_case			)
          if text_pair is None:  # Unpaired text
              if truncation:
                  __lowercase						:     int		      =    text[:, : max_length - 2]  # Allow room for special tokens
              __lowercase      , __lowercase						:     Dict		      =    combine_segments(
                  (text,)			,    start_of_sequence_id=self.cls_token_id			,    end_of_segment_id=self.sep_token_id			)
          else:  # Paired text
              __lowercase						:     int		      =    self.unpaired_tokenize(_snake_case			)
              if truncation:
                  __lowercase      , __lowercase						:     Union[str, Any]		      =    self.paired_trimmer.trim([text, text_pair]			)
              __lowercase      , __lowercase						:     int		      =    combine_segments(
                  (text, text_pair)			,    start_of_sequence_id=self.cls_token_id			,    end_of_segment_id=self.sep_token_id			)
          if padding == "longest":
              __lowercase						:     List[Any]		      =    input_ids.bounding_shape(axis=1			)
              if pad_to_multiple_of is not None:
                  # No ceiling division in tensorflow, so we negate floordiv instead
                  __lowercase						:     List[Any]		      =    pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,    _snake_case			))
          else:
              __lowercase						:     str		      =    max_length

          __lowercase      , __lowercase						:     Optional[int]		      =    pad_model_inputs(_snake_case			,    max_seq_length=_snake_case			,    pad_value=self.pad_token_id			)
          __lowercase						:     Dict		      =    {'''input_ids''': input_ids}
          if return_attention_mask:
              __lowercase						:     Any		      =    attention_mask
          if return_token_type_ids:
              __lowercase      , __lowercase						:     List[str]		      =    pad_model_inputs(
                  _snake_case			,    max_seq_length=_snake_case			,    pad_value=self.pad_token_id			)
              __lowercase						:     Union[str, Any]		      =    token_type_ids
          return output




      def 						snake_case_     ( self						:    str			):
          return {
              "vocab_list": self.vocab_list,
              "do_lower_case": self.do_lower_case,
              "cls_token_id": self.cls_token_id,
              "sep_token_id": self.sep_token_id,
              "pad_token_id": self.pad_token_id,
          }


