
'''simple docstring'''

import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 				lowercase (					__magic_name__			,   __magic_name__			,   __magic_name__=0							):



					'''simple docstring'''







					if name is None:
										UpperCAmelCase		:							List[str]        =       None
					else:
										UpperCAmelCase		:							str        =       "." * max(0			,   spaces - 2							) + "# {:" + str(50 - spaces							) + "s}"
										UpperCAmelCase		:							int        =       fmt.format(__magic_name__							)

					# Print and recurse (if needed).
					if isinstance(__magic_name__			,   __magic_name__							):
										if msg is not None:
															print(__magic_name__							)
										for k in val.keys():
															recursive_print(__magic_name__			,   val[k]			,   spaces + 2							)
					elif isinstance(__magic_name__			,   torch.Tensor							):
										print(__magic_name__			,   ":"			,   val.size()							)
					else:
										print(__magic_name__			,   ":"			,   __magic_name__							)





def 				lowercase (					__magic_name__			,   __magic_name__			,   __magic_name__			,   __magic_name__							):



					'''simple docstring'''







					UpperCAmelCase		:							Any        =       param.size()
					# other versions store [num_heads * num_splits * hidden_size, :]
					UpperCAmelCase		:							Optional[int]        =       (num_heads, num_splits, hidden_size) + input_shape[1:]
					UpperCAmelCase		:							Dict        =       param.view(*__magic_name__							)
					UpperCAmelCase		:							Optional[Any]        =       param.transpose(0			,   1							).contiguous()
					UpperCAmelCase		:							Union[str, Any]        =       param.view(*__magic_name__							)
					return param





def 				lowercase (					__magic_name__			,   __magic_name__							):



					'''simple docstring'''







					UpperCAmelCase		:							int        =       config.n_positions
					UpperCAmelCase		:							List[str]        =       config.n_layer
					UpperCAmelCase		:							Dict        =       config.vocab_size
					UpperCAmelCase		:							int        =       config.n_head
					UpperCAmelCase		:							Optional[Any]        =       config.n_embd // config.n_head

					UpperCAmelCase		:							Dict        =       sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
					UpperCAmelCase		:							Optional[int]        =       {
					    "transformer.wte.weight": word_embeddings,
					    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
					    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
					    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
					}

					UpperCAmelCase		:							str        =       "model.language_model.encoder.layers."
					for i in range(__magic_name__							):
										UpperCAmelCase		:							str        =       torch.tril(torch.ones((n_positions, n_positions)			,   dtype=torch.bool							)							)
										UpperCAmelCase		:							str        =       causal_mask.view(1			,   1			,   __magic_name__			,   __magic_name__							)
										UpperCAmelCase		:							int        =       causal_mask
										UpperCAmelCase		:							Optional[Any]        =       torch.tensor(-1e4			,   dtype=torch.bfloataa							)

										UpperCAmelCase		:							Any        =       sd_megatron[F"{pf}{i}.input_layernorm.weight"]
										UpperCAmelCase		:							Dict        =       sd_megatron[F"{pf}{i}.input_layernorm.bias"]

										UpperCAmelCase		:							Optional[int]        =       sd_megatron[F"{pf}{i}.self_attention.query_key_value.weight"]
										UpperCAmelCase		:							int        =       fix_query_key_value_ordering(__magic_name__			,   3			,   __magic_name__			,   __magic_name__							)
										UpperCAmelCase		:							List[str]        =       vala.transpose(0			,   1							).contiguous()

										UpperCAmelCase		:							str        =       sd_megatron[F"{pf}{i}.self_attention.query_key_value.bias"]
										UpperCAmelCase		:							List[str]        =       fix_query_key_value_ordering(__magic_name__			,   3			,   __magic_name__			,   __magic_name__							)
										UpperCAmelCase		:							Any        =       vala

										UpperCAmelCase		:							Any        =       sd_megatron[F"{pf}{i}.self_attention.dense.weight"].transpose(
										    0			,   1							)
										UpperCAmelCase		:							List[Any]        =       sd_megatron[F"{pf}{i}.self_attention.dense.bias"]
										UpperCAmelCase		:							Optional[int]        =       sd_megatron[F"{pf}{i}.post_attention_layernorm.weight"]
										UpperCAmelCase		:							Optional[Any]        =       sd_megatron[F"{pf}{i}.post_attention_layernorm.bias"]
										UpperCAmelCase		:							int        =       sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0			,   1							)
										UpperCAmelCase		:							Union[str, Any]        =       sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.bias"]
										UpperCAmelCase		:							Tuple        =       sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
										    0			,   1							)
										UpperCAmelCase		:							Tuple        =       sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.bias"]

					# For LM head, transformers' wants the matrix to weight embeddings.
					UpperCAmelCase		:							Optional[int]        =       word_embeddings

					return sd_hf





def 				lowercase (					__magic_name__			,   __magic_name__							):



					'''simple docstring'''







					UpperCAmelCase		:							Dict        =       6_4000
					UpperCAmelCase		:							str        =       config_megatron["encoder_seq_length"]
					UpperCAmelCase		:							Any        =       config_megatron["hidden_size"]
					UpperCAmelCase		:							Optional[int]        =       config_megatron["num_layers"]
					UpperCAmelCase		:							str        =       config_megatron["num_attention_heads"]
					UpperCAmelCase		:							Any        =       config_megatron["ffn_hidden_size"]
					UpperCAmelCase		:							int        =       "gelu"
					UpperCAmelCase		:							int        =       0.1
					UpperCAmelCase		:							List[Any]        =       0.1
					UpperCAmelCase		:							List[Any]        =       0.1
					UpperCAmelCase		:							List[str]        =       config_megatron["layernorm_epsilon"]  # 1e-5
					UpperCAmelCase		:							Any        =       config_megatron["init_method_std"]  # 0.02
					UpperCAmelCase		:							int        =       config_megatron["apply_query_key_layer_scaling"]  # True
					UpperCAmelCase		:							List[Any]        =       True
					UpperCAmelCase		:							Optional[Any]        =       True

					# This identifies the 6.7B (7B) model which uses a different tokenizer
					if config_megatron["hidden_size"] == 4096:
										UpperCAmelCase		:							str        =       1  # <|endoftext|>
										UpperCAmelCase		:							str        =       1  # <|endoftext|>
										UpperCAmelCase		:							List[Any]        =       0  # <unk>
					else:
										UpperCAmelCase		:							Optional[Any]        =       2  # <s>
										UpperCAmelCase		:							Union[str, Any]        =       3  # <|endoftext|>
										UpperCAmelCase		:							Tuple        =       0  # <pad>

					return config_hf





def 				lowercase (					__magic_name__							):



					'''simple docstring'''







					print(__magic_name__							)

					UpperCAmelCase		:							Dict        =       args.checkpoint_path
					UpperCAmelCase		:							List[Any]        =       args.save_path
					if isfile(__magic_name__							):
										raise FileNotFoundError(F"ERROR! could not find file {checkpoint_path}"							)

					# Load the model.
					UpperCAmelCase		:							List[str]        =       torch.load(__magic_name__			,   map_location="cpu"							)

					# Load the config.
					UpperCAmelCase		:							Union[str, Any]        =       checkpoint["hyper_parameters"]["cfg"]
					UpperCAmelCase		:							Optional[int]        =       GPTaConfig()
					UpperCAmelCase		:							List[str]        =       copy_config(config_hf=__magic_name__			,   config_megatron=__magic_name__							)
					UpperCAmelCase		:							List[str]        =       ["GPT2LMHeadModel"]

					UpperCAmelCase		:							Union[str, Any]        =       checkpoint["state_dict"]

					# Convert.
					print("Converting"							)
					UpperCAmelCase		:							List[str]        =       convert_megatron_checkpoint(__magic_name__			,   __magic_name__							)

					# Print the structure of converted state dict.
					if args.print_checkpoint_structure:
										recursive_print(__magic_name__			,   __magic_name__							)

					UpperCAmelCase		:							Any        =       "GPTSw3Tokenizer"

					# Store the config to file.
					print("Saving config"							)
					config_hf.save_pretrained(__magic_name__							)

					# Store the state_dict to file.
					UpperCAmelCase		:							Tuple        =       os.path.join(__magic_name__			,   "pytorch_model.bin"							)
					print(F"Saving checkpoint to \"{output_checkpoint_file}\""							)
					torch.save(__magic_name__			,   __magic_name__							)


if __name__ == "__main__":
				a       :      List[Any]  	=			argparse.ArgumentParser()
				parser.add_argument(
				    "--checkpoint_path",
				    type=str,
				    required=True,
				    help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
				)
				parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
				parser.add_argument("--print-checkpoint-structure", action="store_true")
				a       :      Any  	=			parser.parse_args()
				main(_args)



'''simple docstring'''

import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







class   UpperCamelCase__   (       lowercase__       ):


		"""simple docstring"""

		def __init__(			self   ,      snake_case   ,      snake_case = None   ,      snake_case = None   ,      snake_case = None   ,      snake_case = False   ,      snake_case = False   ,      snake_case = None   ,      **snake_case   ,      ):


							'''simple docstring'''

							super().__init__(
							    snake_case   ,      split=snake_case   ,      features=snake_case   ,      cache_dir=snake_case   ,      keep_in_memory=snake_case   ,      streaming=snake_case   ,      num_proc=snake_case   ,      **snake_case   ,      )
							UpperCAmelCase		:							str        =       path_or_paths if isinstance(snake_case   ,      snake_case      ) else {self.split: path_or_paths}
							UpperCAmelCase		:							int        =       Csv(
							    cache_dir=snake_case   ,      data_files=snake_case   ,      features=snake_case   ,      **snake_case   ,      )







		def 				A_	(			self      ):


							'''simple docstring'''

							if self.streaming:
												UpperCAmelCase		:							Optional[Any]        =       self.builder.as_streaming_dataset(split=self.split      )
							# Build regular (map-style) dataset
							else:
												UpperCAmelCase		:							Union[str, Any]        =       None
												UpperCAmelCase		:							str        =       None
												UpperCAmelCase		:							Union[str, Any]        =       None
												UpperCAmelCase		:							int        =       None

												self.builder.download_and_prepare(
												    download_config=snake_case   ,      download_mode=snake_case   ,      verification_mode=snake_case   ,      base_path=snake_case   ,      num_proc=self.num_proc   ,      )
												UpperCAmelCase		:							Tuple        =       self.builder.as_dataset(
												    split=self.split   ,      verification_mode=snake_case   ,      in_memory=self.keep_in_memory      )
							return dataset







class   UpperCamelCase__   :


		"""simple docstring"""

		def __init__(			self   ,      snake_case   ,      snake_case   ,      snake_case = None   ,      snake_case = None   ,      **snake_case   ,      ):


							'''simple docstring'''

							if num_proc is not None and num_proc <= 0:
												raise ValueError(f"num_proc {num_proc} must be an integer > 0."      )

							UpperCAmelCase		:							Tuple        =       dataset
							UpperCAmelCase		:							Dict        =       path_or_buf
							UpperCAmelCase		:							List[Any]        =       batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
							UpperCAmelCase		:							str        =       num_proc
							UpperCAmelCase		:							List[Any]        =       "utf-8"
							UpperCAmelCase		:							Tuple        =       to_csv_kwargs







		def 				A_	(			self      ):


							'''simple docstring'''

							UpperCAmelCase		:							List[str]        =       self.to_csv_kwargs.pop("path_or_buf"   ,      snake_case      )
							UpperCAmelCase		:							Any        =       self.to_csv_kwargs.pop("header"   ,      snake_case      )
							UpperCAmelCase		:							Dict        =       self.to_csv_kwargs.pop("index"   ,      snake_case      )

							if isinstance(self.path_or_buf   ,      (str, bytes, os.PathLike)      ):
												with open(self.path_or_buf   ,      "wb+"      ) as buffer:
																	UpperCAmelCase		:							Tuple        =       self._write(file_obj=snake_case   ,      header=snake_case   ,      index=snake_case   ,      **self.to_csv_kwargs      )
							else:
												UpperCAmelCase		:							Any        =       self._write(file_obj=self.path_or_buf   ,      header=snake_case   ,      index=snake_case   ,      **self.to_csv_kwargs      )
							return written







		def 				A_	(			self   ,      snake_case      ):


							'''simple docstring'''

							UpperCAmelCase ,			UpperCAmelCase ,			UpperCAmelCase ,			UpperCAmelCase		:							Optional[int]        =       args

							UpperCAmelCase		:							List[str]        =       query_table(
							    table=self.dataset.data   ,      key=slice(snake_case   ,      offset + self.batch_size      )   ,      indices=self.dataset._indices   ,      )
							UpperCAmelCase		:							int        =       batch.to_pandas().to_csv(
							    path_or_buf=snake_case   ,      header=header if (offset == 0) else False   ,      index=snake_case   ,      **snake_case      )
							return csv_str.encode(self.encoding      )







		def 				A_	(			self   ,      snake_case   ,      snake_case   ,      snake_case   ,      **snake_case      ):


							'''simple docstring'''

							UpperCAmelCase		:							List[str]        =       0

							if self.num_proc is None or self.num_proc == 1:
												for offset in logging.tqdm(
												    range(0   ,      len(self.dataset      )   ,      self.batch_size      )   ,      unit="ba"   ,      disable=not logging.is_progress_bar_enabled()   ,      desc="Creating CSV from Arrow format"   ,      ):
																	UpperCAmelCase		:							Any        =       self._batch_csv((offset, header, index, to_csv_kwargs)      )
																	written += file_obj.write(snake_case      )

							else:
												UpperCAmelCase ,			UpperCAmelCase		:							Union[str, Any]        =       len(self.dataset      ), self.batch_size
												with multiprocessing.Pool(self.num_proc      ) as pool:
																	for csv_str in logging.tqdm(
																	    pool.imap(
																	        self._batch_csv   ,      [(offset, header, index, to_csv_kwargs) for offset in range(0   ,      snake_case   ,      snake_case      )]   ,      )   ,      total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,      unit="ba"   ,      disable=not logging.is_progress_bar_enabled()   ,      desc="Creating CSV from Arrow format"   ,      ):
																						written += file_obj.write(snake_case      )

							return written
