

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig
def 					a				(      A__     :       Union[str, Any]				,       A__     :       str				,       A__     :       Union[str, Any]=0					)   ->				Tuple:





     """simple docstring"""



     if name is None:
          _lowercase     	=None
     else:
          _lowercase     	="." * max(0				,       spaces - 2					) + "# {:" + str(50 - spaces					) + "s}"
          _lowercase     	=fmt.format(_a					)

     # Print and recurse (if needed).
     if isinstance(_a				,       _a					):
          if msg is not None:
               print(_a					)
          for k in val.keys():
               recursive_print(_a				,       val[k]				,       spaces + 2					)
     elif isinstance(_a				,       torch.Tensor					):
          print(_a				,       ':'				,       val.size()					)
     else:
          print(_a				,       ':'				,       _a					)
def 					a				(      A__     :       List[str]				,       A__     :       Any				,       A__     :       str				,       A__     :       int				,       A__     :       Tuple					)   ->				Union[str, Any]:





     """simple docstring"""



     _lowercase     	=param.size()
     if checkpoint_version == 1.0:
          # version 1.0 stores [num_heads * hidden_size * num_splits, :]
          _lowercase     	=(num_heads, hidden_size, num_splits) + input_shape[1:]
          _lowercase     	=param.view(*_a					)
          _lowercase     	=param.transpose(0				,       2					)
          _lowercase     	=param.transpose(1				,       2					).contiguous()
     elif checkpoint_version >= 2.0:
          # other versions store [num_heads * num_splits * hidden_size, :]
          _lowercase     	=(num_heads, num_splits, hidden_size) + input_shape[1:]
          _lowercase     	=param.view(*_a					)
          _lowercase     	=param.transpose(0				,       1					).contiguous()
     _lowercase     	=param.view(*_a					)
     return param
def 					a				(      A__     :       int				,       A__     :       int				,       A__     :       int					)   ->				List[str]:





     """simple docstring"""



     _lowercase     	={}

     # old versions did not store training args
     _lowercase     	=input_state_dict.get('args'				,       _a					)
     if ds_args is not None:
          # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
          # from pprint import pprint
          # pprint(vars(ds_args))

          _lowercase     	=ds_args.tokenizer_type
          _lowercase     	=ds_args.padded_vocab_size
          _lowercase     	=ds_args.max_position_embeddings
          _lowercase     	=ds_args.hidden_size
          _lowercase     	=ds_args.num_layers
          _lowercase     	=ds_args.num_attention_heads
          _lowercase     	=ds_args.ffn_hidden_size if "ffn_hidden_size" in ds_args else 4 * ds_args.hidden_size
          # pprint(config)

     # The number of heads.
     _lowercase     	=config.num_attention_heads
     # The hidden_size per head.
     _lowercase     	=config.hidden_size // heads
     # Megatron-LM checkpoint version
     if "checkpoint_version" in input_state_dict.keys():
          _lowercase     	=input_state_dict["checkpoint_version"]
     else:
          _lowercase     	=0.0

     # The model.
     _lowercase     	=input_state_dict["model"]
     # The language model.
     _lowercase     	=model["language_model"]
     # The embeddings.
     _lowercase     	=lm["embedding"]

     # The word embeddings.
     _lowercase     	=embeddings["word_embeddings"]["weight"]
     # Truncate the embedding table to vocab_size rows.
     _lowercase     	=word_embeddings[: config.vocab_size, :]
     # Store the word embeddings.
     _lowercase     	=word_embeddings

     # The position embeddings.
     _lowercase     	=embeddings["position_embeddings"]["weight"]
     assert pos_embeddings.size(0					) == config.max_position_embeddings and pos_embeddings.size(1					) == config.hidden_size
     # Store the position embeddings.
     _lowercase     	=pos_embeddings

     # The token-type embeddings.
     _lowercase     	=embeddings["tokentype_embeddings"]["weight"]
     # Store the position embeddings.
     _lowercase     	=tokentype_embeddings

     # The transformer.
     _lowercase     	=lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

     # The regex to extract layer names.
     _lowercase     	=re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'					)

     # The simple map of names for "automated" rules.
     _lowercase     	={
         "attention.dense": ".attention.output.dense.",
         "self_attention.dense": ".attention.output.dense.",
         "mlp.dense_h_to_4h": ".intermediate.dense.",
         "mlp.dense_4h_to_h": ".output.dense.",
     }

     # Keep track of the attention/query/value tensor.
     _lowercase     	=None

     # Extract the layers.
     for key, val in transformer.items():
          # Match the name.
          _lowercase     	=layer_re.match(_a					)

          # Stop if that's not a layer
          if m is None:
               break

          # The index of the layer.
          _lowercase     	=int(m.group(1					)					)
          # The name of the operation.
          _lowercase     	=m.group(2					)
          # Is it a weight or a bias?
          _lowercase     	=m.group(3					)

          # The name of the layer.
          _lowercase     	=F'''bert.encoder.layer.{layer_idx}'''

          # For layernorm(s), simply store the layer norm.
          if op_name.endswith('layernorm'					):
               _lowercase     	="attention.ln" if op_name.startswith('input'					) else "ln"
               _lowercase     	=val

          # Transpose the QKV matrix.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "weight":
               # Make sure the QKV pointer is nil.
               assert attention_qkv_weight is None, ""

               _lowercase     	=fix_query_key_value_ordering(_a				,       _a				,       3				,       _a				,       _a					)
               # Store the tensor as we need the bias as well to interleave QKV and biases.
               _lowercase     	=out_val

          # Transpose the bias.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "bias":
               # Make sure we read the weight tensor.
               assert attention_qkv_weight is not None, ""

               # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
               _lowercase     	=attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
               _lowercase     	=attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
               _lowercase     	=attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

               _lowercase     	=fix_query_key_value_ordering(_a				,       _a				,       3				,       _a				,       _a					)
               # Split the bias.
               _lowercase     	=out_val[0 * config.hidden_size : 1 * config.hidden_size]
               _lowercase     	=out_val[1 * config.hidden_size : 2 * config.hidden_size]
               _lowercase     	=out_val[2 * config.hidden_size : 3 * config.hidden_size]

               # Store.
               _lowercase     	=q
               _lowercase     	=q_bias
               _lowercase     	=k
               _lowercase     	=k_bias
               _lowercase     	=v
               _lowercase     	=v_bias

               # Clear the stored tensor.
               _lowercase     	=None

          # Copy weights and biases as is.
          elif weight_or_bias in ["weight", "bias"]:
               _lowercase     	=megatron_to_transformers[op_name]
               _lowercase     	=val

    # The final layernorm.
     _lowercase     	=transformer["final_layernorm.weight"]
     _lowercase     	=transformer["final_layernorm.bias"]

     # The pooler.
     _lowercase     	=lm["pooler"]

     # Store the matrix and the bias.
     _lowercase     	=pooler["dense.weight"]
     _lowercase     	=pooler["dense.bias"]

     # The LM head from Megatron (for RACE).
     _lowercase     	=model["lm_head"]

     # The transform matrix.
     _lowercase     	=lm_head["dense.weight"]
     _lowercase     	=lm_head["dense.bias"]

     # The transform LN.
     _lowercase     	=lm_head["layernorm.weight"]
     _lowercase     	=lm_head["layernorm.bias"]

     # For the decoder, we replicate the weights.
     _lowercase     	=word_embeddings
     _lowercase     	=lm_head["bias"]

     # The classifier from Megatron (for MLNI).
     _lowercase     	=model["binary_head"]

     # Store the classifier.
     _lowercase     	=binary_head["weight"]
     _lowercase     	=binary_head["bias"]

     # It should be done!
     return output_state_dict
def 					a				(      )   ->				List[str]:





     """simple docstring"""



     _lowercase     	=argparse.ArgumentParser()
     parser.add_argument('--print-checkpoint-structure'				,       action='store_true'					)
     parser.add_argument('path_to_checkpoint'				,       type=_a				,       help='Path to the ZIP file containing the checkpoint'					)
     parser.add_argument(
         '--config_file'				,       default=''				,       type=_a				,       help='An optional config json file describing the pre-trained model.'				,       )
     _lowercase     	=parser.parse_args()

     # Extract the basename.
     _lowercase     	=os.path.dirname(args.path_to_checkpoint					)

     # Load the model.
     # the .zip is very optional, let's keep it for backward compatibility
     print(F'''Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\"'''					)
     if args.path_to_checkpoint.endswith('.zip'					):
          with zipfile.ZipFile(args.path_to_checkpoint				,       'r'					) as checkpoint:
               with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'					) as pytorch_dict:
                    _lowercase     	=torch.load(_a				,       map_location='cpu'					)
     else:
          _lowercase     	=torch.load(args.path_to_checkpoint				,       map_location='cpu'					)

     if args.config_file == "":
          # Default config of megatron-bert 345m
          _lowercase     	=MegatronBertConfig()

          # different megatron-bert-*-345m models have different vocab sizes, so override the default
          # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
          _lowercase     	=input_state_dict["model"]["lm_head"]["bias"].numel()
     else:
          _lowercase     	=MegatronBertConfig.from_json_file(args.config_file					)

     # Convert.
     print('Converting'					)
     _lowercase     	=convert_megatron_checkpoint(_a				,       _a				,       _a					)

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(_a				,       _a					)

     # Store the config to file.
     print('Saving config'					)
     config.save_pretrained(_a					)

     # Store the state_dict to file.
     _lowercase     	=os.path.join(_a				,       'pytorch_model.bin'					)
     print(F'''Saving checkpoint to \"{output_checkpoint_file}\"'''					)
     torch.save(_a				,       _a					)


####################################################################################################

if __name__ == "__main__":
  main()

####################################################################################################





import json
import os
import shutil
import tempfile
import unittest

from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor, WavaVecaProcessor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_wavaveca import floats_list





class 			__lowerCAmelCase					( unittest.TestCase							):




  def   A__					(    self )  ->			Tuple:
        '''simple docstring'''






        _lowercase     	='<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K \' X J Q Z'.split(' ' )
        _lowercase     	=dict(zip(lowerCAmelCase							,       range(len(lowerCAmelCase ) ) ) )

        _lowercase     	={
            'pad_token': '<pad>',
            'unk_token': '<unk>',
            'bos_token': '<s>',
            'eos_token': '</s>',
        }
        _lowercase     	={
            'feature_size': 1,
            'padding_value': 0.0,
            'sampling_rate': 16_000,
            'return_attention_mask': False,
            'do_normalize': True,
        }

        _lowercase     	=tempfile.mkdtemp()
        _lowercase     	=os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['vocab_file'] )
        _lowercase     	=os.path.join(self.tmpdirname							,       lowerCAmelCase )
        with open(self.vocab_file							,       'w'							,       encoding='utf-8' ) as fp:
              fp.write(json.dumps(lowerCAmelCase ) + '\n' )

        with open(self.feature_extraction_file							,       'w'							,       encoding='utf-8' ) as fp:
              fp.write(json.dumps(lowerCAmelCase ) + '\n' )




  def   A__					(    self							,       **lowerCAmelCase )  ->			Optional[Any]:
        '''simple docstring'''






        _lowercase     	=self.add_kwargs_tokens_map.copy()
        kwargs.update(lowerCAmelCase )
        return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname							,       **lowerCAmelCase )




  def   A__					(    self							,       **lowerCAmelCase )  ->			List[str]:
        '''simple docstring'''






        return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname							,       **lowerCAmelCase )




  def   A__					(    self )  ->			Union[str, Any]:
        '''simple docstring'''






        shutil.rmtree(self.tmpdirname )




  def   A__					(    self )  ->			Optional[int]:
        '''simple docstring'''






        _lowercase     	=self.get_tokenizer()
        _lowercase     	=self.get_feature_extractor()

        _lowercase     	=WavaVecaProcessor(tokenizer=lowerCAmelCase							,       feature_extractor=lowerCAmelCase )

        processor.save_pretrained(self.tmpdirname )
        _lowercase     	=WavaVecaProcessor.from_pretrained(self.tmpdirname )

        self.assertEqual(processor.tokenizer.get_vocab()							,       tokenizer.get_vocab() )
        self.assertIsInstance(processor.tokenizer							,       lowerCAmelCase )

        self.assertEqual(processor.feature_extractor.to_json_string()							,       feature_extractor.to_json_string() )
        self.assertIsInstance(processor.feature_extractor							,       lowerCAmelCase )




  def   A__					(    self )  ->			str:
        '''simple docstring'''






        _lowercase     	=WavaVecaProcessor(tokenizer=self.get_tokenizer()							,       feature_extractor=self.get_feature_extractor() )
        processor.save_pretrained(self.tmpdirname )

        _lowercase     	=self.get_tokenizer(bos_token='(BOS)'							,       eos_token='(EOS)' )
        _lowercase     	=self.get_feature_extractor(do_normalize=lowerCAmelCase							,       padding_value=1.0 )

        _lowercase     	=WavaVecaProcessor.from_pretrained(
            self.tmpdirname							,       bos_token='(BOS)'							,       eos_token='(EOS)'							,       do_normalize=lowerCAmelCase							,       padding_value=1.0 )

        self.assertEqual(processor.tokenizer.get_vocab()							,       tokenizer_add_kwargs.get_vocab() )
        self.assertIsInstance(processor.tokenizer							,       lowerCAmelCase )

        self.assertEqual(processor.feature_extractor.to_json_string()							,       feature_extractor_add_kwargs.to_json_string() )
        self.assertIsInstance(processor.feature_extractor							,       lowerCAmelCase )




  def   A__					(    self )  ->			List[Any]:
        '''simple docstring'''






        _lowercase     	=self.get_feature_extractor()
        _lowercase     	=self.get_tokenizer()

        _lowercase     	=WavaVecaProcessor(tokenizer=lowerCAmelCase							,       feature_extractor=lowerCAmelCase )

        _lowercase     	=floats_list((3, 1_000) )

        _lowercase     	=feature_extractor(lowerCAmelCase							,       return_tensors='np' )
        _lowercase     	=processor(lowerCAmelCase							,       return_tensors='np' )

        for key in input_feat_extract.keys():
              self.assertAlmostEqual(input_feat_extract[key].sum()							,       input_processor[key].sum()							,       delta=1e-2 )




  def   A__					(    self )  ->			int:
        '''simple docstring'''






        _lowercase     	=self.get_feature_extractor()
        _lowercase     	=self.get_tokenizer()

        _lowercase     	=WavaVecaProcessor(tokenizer=lowerCAmelCase							,       feature_extractor=lowerCAmelCase )

        _lowercase     	='This is a test string'

        _lowercase     	=processor(text=lowerCAmelCase )

        _lowercase     	=tokenizer(lowerCAmelCase )

        for key in encoded_tok.keys():
              self.assertListEqual(encoded_tok[key]							,       encoded_processor[key] )




  def   A__					(    self )  ->			str:
        '''simple docstring'''






        _lowercase     	=self.get_feature_extractor()
        _lowercase     	=self.get_tokenizer()

        _lowercase     	=WavaVecaProcessor(tokenizer=lowerCAmelCase							,       feature_extractor=lowerCAmelCase )

        _lowercase     	=[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        _lowercase     	=processor.batch_decode(lowerCAmelCase )
        _lowercase     	=tokenizer.batch_decode(lowerCAmelCase )

        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )




  def   A__					(    self )  ->			str:
        '''simple docstring'''






        _lowercase     	=self.get_feature_extractor()
        _lowercase     	=self.get_tokenizer()

        _lowercase     	=WavaVecaProcessor(tokenizer=lowerCAmelCase							,       feature_extractor=lowerCAmelCase )

        self.assertListEqual(
            processor.model_input_names							,       feature_extractor.model_input_names							,       msg='`processor` and `feature_extractor` model input names do not match'							,       )


