# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig







def       snake_case__    (       lowerCAmelCase_,   lowerCAmelCase_,   lowerCAmelCase_=0      ):





     """simple docstring"""







     if name is None:
          SCREAMING_SNAKE_CASE										=None
     else:
          SCREAMING_SNAKE_CASE										='.' * max(0,   spaces - 2      ) + '# {:' + str(50 - spaces      ) + 's}'
          SCREAMING_SNAKE_CASE										=fmt.format(lowercase_      )

     # Print and recurse (if needed).
     if isinstance(lowercase_,   lowercase_      ):
          if msg is not None:
               print(lowercase_      )
          for k in val.keys():
               recursive_print(lowercase_,   val[k],   spaces + 2      )
     elif isinstance(lowercase_,   torch.Tensor      ):
          print(lowercase_,   ':',   val.size()      )
     else:
          print(lowercase_,   ':',   lowercase_      )







def       snake_case__    (       lowerCAmelCase_,   lowerCAmelCase_,   lowerCAmelCase_,   lowerCAmelCase_,   lowerCAmelCase_      ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE										=param.size()
     if checkpoint_version == 1.0:
          # version 1.0 stores [num_heads * hidden_size * num_splits, :]
          SCREAMING_SNAKE_CASE										=(num_heads, hidden_size, num_splits) + input_shape[1:]
          SCREAMING_SNAKE_CASE										=param.view(*lowercase_      )
          SCREAMING_SNAKE_CASE										=param.transpose(0,   2      )
          SCREAMING_SNAKE_CASE										=param.transpose(1,   2      ).contiguous()
     elif checkpoint_version >= 2.0:
          # other versions store [num_heads * num_splits * hidden_size, :]
          SCREAMING_SNAKE_CASE										=(num_heads, num_splits, hidden_size) + input_shape[1:]
          SCREAMING_SNAKE_CASE										=param.view(*lowercase_      )
          SCREAMING_SNAKE_CASE										=param.transpose(0,   1      ).contiguous()
     SCREAMING_SNAKE_CASE										=param.view(*lowercase_      )
     return param







def       snake_case__    (       lowerCAmelCase_,   lowerCAmelCase_,   lowerCAmelCase_      ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE										={}

     # old versions did not store training args
     SCREAMING_SNAKE_CASE										=input_state_dict.get('args',   lowercase_      )
     if ds_args is not None:
          # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
          # from pprint import pprint
          # pprint(vars(ds_args))

          SCREAMING_SNAKE_CASE										=ds_args.tokenizer_type
          SCREAMING_SNAKE_CASE										=ds_args.padded_vocab_size
          SCREAMING_SNAKE_CASE										=ds_args.max_position_embeddings
          SCREAMING_SNAKE_CASE										=ds_args.hidden_size
          SCREAMING_SNAKE_CASE										=ds_args.num_layers
          SCREAMING_SNAKE_CASE										=ds_args.num_attention_heads
          SCREAMING_SNAKE_CASE										=ds_args.ffn_hidden_size if 'ffn_hidden_size' in ds_args else 4 * ds_args.hidden_size
          # pprint(config)

     # The number of heads.
     SCREAMING_SNAKE_CASE										=config.num_attention_heads
     # The hidden_size per head.
     SCREAMING_SNAKE_CASE										=config.hidden_size // heads
     # Megatron-LM checkpoint version
     if "checkpoint_version" in input_state_dict.keys():
          SCREAMING_SNAKE_CASE										=input_state_dict['checkpoint_version']
     else:
          SCREAMING_SNAKE_CASE										=0.0

     # The model.
     SCREAMING_SNAKE_CASE										=input_state_dict['model']
     # The language model.
     SCREAMING_SNAKE_CASE										=model['language_model']
     # The embeddings.
     SCREAMING_SNAKE_CASE										=lm['embedding']

     # The word embeddings.
     SCREAMING_SNAKE_CASE										=embeddings['word_embeddings']['weight']
     # Truncate the embedding table to vocab_size rows.
     SCREAMING_SNAKE_CASE										=word_embeddings[: config.vocab_size, :]
     # Store the word embeddings.
     SCREAMING_SNAKE_CASE										=word_embeddings

     # The position embeddings.
     SCREAMING_SNAKE_CASE										=embeddings['position_embeddings']['weight']
     assert pos_embeddings.size(0      ) == config.max_position_embeddings and pos_embeddings.size(1      ) == config.hidden_size
     # Store the position embeddings.
     SCREAMING_SNAKE_CASE										=pos_embeddings

     # The token-type embeddings.
     SCREAMING_SNAKE_CASE										=embeddings['tokentype_embeddings']['weight']
     # Store the position embeddings.
     SCREAMING_SNAKE_CASE										=tokentype_embeddings

     # The transformer.
     SCREAMING_SNAKE_CASE										=lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

     # The regex to extract layer names.
     SCREAMING_SNAKE_CASE										=re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'      )

     # The simple map of names for "automated" rules.
     SCREAMING_SNAKE_CASE										={
         'attention.dense': '.attention.output.dense.',
         'self_attention.dense': '.attention.output.dense.',
         'mlp.dense_h_to_4h': '.intermediate.dense.',
         'mlp.dense_4h_to_h': '.output.dense.',
     }

     # Keep track of the attention/query/value tensor.
     SCREAMING_SNAKE_CASE										=None

     # Extract the layers.
     for key, val in transformer.items():
          # Match the name.
          SCREAMING_SNAKE_CASE										=layer_re.match(lowercase_      )

          # Stop if that's not a layer
          if m is None:
               break

          # The index of the layer.
          SCREAMING_SNAKE_CASE										=int(m.group(1      )      )
          # The name of the operation.
          SCREAMING_SNAKE_CASE										=m.group(2      )
          # Is it a weight or a bias?
          SCREAMING_SNAKE_CASE										=m.group(3      )

          # The name of the layer.
          SCREAMING_SNAKE_CASE										=F'bert.encoder.layer.{layer_idx}'

          # For layernorm(s), simply store the layer norm.
          if op_name.endswith('layernorm'      ):
               SCREAMING_SNAKE_CASE										='attention.ln' if op_name.startswith('input'      ) else 'ln'
               SCREAMING_SNAKE_CASE										=val

          # Transpose the QKV matrix.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "weight":
               # Make sure the QKV pointer is nil.
               assert attention_qkv_weight is None, ""

               SCREAMING_SNAKE_CASE										=fix_query_key_value_ordering(lowercase_,   lowercase_,   3,   lowercase_,   lowercase_      )
               # Store the tensor as we need the bias as well to interleave QKV and biases.
               SCREAMING_SNAKE_CASE										=out_val

          # Transpose the bias.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "bias":
               # Make sure we read the weight tensor.
               assert attention_qkv_weight is not None, ""

               # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
               SCREAMING_SNAKE_CASE										=attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
               SCREAMING_SNAKE_CASE										=attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
               SCREAMING_SNAKE_CASE										=attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

               SCREAMING_SNAKE_CASE										=fix_query_key_value_ordering(lowercase_,   lowercase_,   3,   lowercase_,   lowercase_      )
               # Split the bias.
               SCREAMING_SNAKE_CASE										=out_val[0 * config.hidden_size : 1 * config.hidden_size]
               SCREAMING_SNAKE_CASE										=out_val[1 * config.hidden_size : 2 * config.hidden_size]
               SCREAMING_SNAKE_CASE										=out_val[2 * config.hidden_size : 3 * config.hidden_size]

               # Store.
               SCREAMING_SNAKE_CASE										=q
               SCREAMING_SNAKE_CASE										=q_bias
               SCREAMING_SNAKE_CASE										=k
               SCREAMING_SNAKE_CASE										=k_bias
               SCREAMING_SNAKE_CASE										=v
               SCREAMING_SNAKE_CASE										=v_bias

               # Clear the stored tensor.
               SCREAMING_SNAKE_CASE										=None

          # Copy weights and biases as is.
          elif weight_or_bias in ["weight", "bias"]:
               SCREAMING_SNAKE_CASE										=megatron_to_transformers[op_name]
               SCREAMING_SNAKE_CASE										=val

                # The final layernorm.
     SCREAMING_SNAKE_CASE										=transformer['final_layernorm.weight']
     SCREAMING_SNAKE_CASE										=transformer['final_layernorm.bias']

     # The pooler.
     SCREAMING_SNAKE_CASE										=lm['pooler']

     # Store the matrix and the bias.
     SCREAMING_SNAKE_CASE										=pooler['dense.weight']
     SCREAMING_SNAKE_CASE										=pooler['dense.bias']

     # The LM head from Megatron (for RACE).
     SCREAMING_SNAKE_CASE										=model['lm_head']

     # The transform matrix.
     SCREAMING_SNAKE_CASE										=lm_head['dense.weight']
     SCREAMING_SNAKE_CASE										=lm_head['dense.bias']

     # The transform LN.
     SCREAMING_SNAKE_CASE										=lm_head['layernorm.weight']
     SCREAMING_SNAKE_CASE										=lm_head['layernorm.bias']

     # For the decoder, we replicate the weights.
     SCREAMING_SNAKE_CASE										=word_embeddings
     SCREAMING_SNAKE_CASE										=lm_head['bias']

     # The classifier from Megatron (for MLNI).
     SCREAMING_SNAKE_CASE										=model['binary_head']

     # Store the classifier.
     SCREAMING_SNAKE_CASE										=binary_head['weight']
     SCREAMING_SNAKE_CASE										=binary_head['bias']

     # It should be done!
     return output_state_dict







def       snake_case__    (       ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE										=argparse.ArgumentParser()
     parser.add_argument('--print-checkpoint-structure',   action='store_true'      )
     parser.add_argument('path_to_checkpoint',   type=lowercase_,   help='Path to the ZIP file containing the checkpoint'      )
     parser.add_argument(
         '--config_file',   default='',   type=lowercase_,   help='An optional config json file describing the pre-trained model.',   )
     SCREAMING_SNAKE_CASE										=parser.parse_args()

     # Extract the basename.
     SCREAMING_SNAKE_CASE										=os.path.dirname(args.path_to_checkpoint      )

     # Load the model.
     # the .zip is very optional, let's keep it for backward compatibility
     print(F'Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'      )
     if args.path_to_checkpoint.endswith('.zip'      ):
          with zipfile.ZipFile(args.path_to_checkpoint,   'r'      ) as checkpoint:
               with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'      ) as pytorch_dict:
                    SCREAMING_SNAKE_CASE										=torch.load(lowercase_,   map_location='cpu'      )
     else:
          SCREAMING_SNAKE_CASE										=torch.load(args.path_to_checkpoint,   map_location='cpu'      )

     if args.config_file == "":
          # Default config of megatron-bert 345m
          SCREAMING_SNAKE_CASE										=MegatronBertConfig()

          # different megatron-bert-*-345m models have different vocab sizes, so override the default
          # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
          SCREAMING_SNAKE_CASE										=input_state_dict['model']['lm_head']['bias'].numel()
     else:
          SCREAMING_SNAKE_CASE										=MegatronBertConfig.from_json_file(args.config_file      )

     # Convert.
     print('Converting'      )
     SCREAMING_SNAKE_CASE										=convert_megatron_checkpoint(lowercase_,   lowercase_,   lowercase_      )

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(lowercase_,   lowercase_      )

     # Store the config to file.
     print('Saving config'      )
     config.save_pretrained(lowercase_      )

     # Store the state_dict to file.
     SCREAMING_SNAKE_CASE										=os.path.join(lowercase_,   'pytorch_model.bin'      )
     print(F'Saving checkpoint to "{output_checkpoint_file}"'      )
     torch.save(lowercase_,   lowercase_      )


####################################################################################################

if __name__ == "__main__":
 main()

####################################################################################################




import gc
import unittest

import numpy as np
import torch

from diffusers import RePaintPipeline, RePaintScheduler, UNetaDModel
from diffusers.utils.testing_utils import (
    enable_full_determinism,
    load_image,
    load_numpy,
    nightly,
    require_torch_gpu,
    skip_mps,
    torch_device,
)

from ..pipeline_params import IMAGE_INPAINTING_BATCH_PARAMS, IMAGE_INPAINTING_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()



class 		a_	(							lowerCamelCase_			,      unittest.TestCase					):



      """simple docstring"""







      __UpperCAmelCase      		=				RePaintPipeline
      __UpperCAmelCase      		=				IMAGE_INPAINTING_PARAMS - {'width', 'height', 'guidance_scale'}
      __UpperCAmelCase      		=				PipelineTesterMixin.required_optional_params - {
          'latents',
          'num_images_per_prompt',
          'callback',
          'callback_steps',
      }
      __UpperCAmelCase      		=				IMAGE_INPAINTING_BATCH_PARAMS
      __UpperCAmelCase      		=				False
      def 	_lowerCAmelCase					(				self					:							Tuple    ):
           torch.manual_seed(0    )
           torch.manual_seed(0    )
           SCREAMING_SNAKE_CASE										=UNetaDModel(
               block_out_channels=(32, 64)      ,layers_per_block=2      ,sample_size=32      ,in_channels=3      ,out_channels=3      ,down_block_types=('DownBlock2D', 'AttnDownBlock2D')      ,up_block_types=('AttnUpBlock2D', 'UpBlock2D')      ,)
           SCREAMING_SNAKE_CASE										=RePaintScheduler()
           SCREAMING_SNAKE_CASE										={'unet': unet, 'scheduler': scheduler}
           return components
      def 	_lowerCAmelCase					(				self					:							Tuple      ,snake_case					:							Optional[int]      ,snake_case					:							Dict=0    ):
           if str(snake_case    ).startswith('mps'    ):
                SCREAMING_SNAKE_CASE										=torch.manual_seed(snake_case    )
           else:
                SCREAMING_SNAKE_CASE										=torch.Generator(device=snake_case    ).manual_seed(snake_case    )
           SCREAMING_SNAKE_CASE										=np.random.RandomState(snake_case    ).standard_normal((1, 3, 32, 32)    )
           SCREAMING_SNAKE_CASE										=torch.from_numpy(snake_case    ).to(device=snake_case      ,dtype=torch.floataa    )
           SCREAMING_SNAKE_CASE										=(image > 0).to(device=snake_case      ,dtype=torch.floataa    )
           SCREAMING_SNAKE_CASE										={
               'image': image,
               'mask_image': mask,
               'generator': generator,
               'num_inference_steps': 5,
               'eta': 0.0,
               'jump_length': 2,
               'jump_n_sample': 2,
               'output_type': 'numpy',
           }
           return inputs
      def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
           SCREAMING_SNAKE_CASE										='cpu'  # ensure determinism for the device-dependent torch.Generator
           SCREAMING_SNAKE_CASE										=self.get_dummy_components()
           SCREAMING_SNAKE_CASE										=RePaintPipeline(**snake_case    )
           SCREAMING_SNAKE_CASE										=sd_pipe.to(snake_case    )
           sd_pipe.set_progress_bar_config(disable=snake_case    )

           SCREAMING_SNAKE_CASE										=self.get_dummy_inputs(snake_case    )
           SCREAMING_SNAKE_CASE										=sd_pipe(**snake_case    ).images
           SCREAMING_SNAKE_CASE										=image[0, -3:, -3:, -1]

           assert image.shape == (1, 32, 32, 3)
           SCREAMING_SNAKE_CASE										=np.array([1.0_000, 0.5_426, 0.5_497, 0.2_200, 1.0_000, 1.0_000, 0.5_623, 1.0_000, 0.6_274]    )

           assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-3
      @skip_mps
      def 	_lowerCAmelCase					(				self					:							Optional[int]    ):
           return super().test_save_load_local()
      @unittest.skip('non-deterministic pipeline'    )
      def 	_lowerCAmelCase					(				self					:							Tuple    ):
           return super().test_inference_batch_single_identical()
      @skip_mps
      def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
           return super().test_dict_tuple_outputs_equivalent()
      @skip_mps
      def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
           return super().test_save_load_optional_components()


      @skip_mps
      def 	_lowerCAmelCase					(				self					:							Any    ):
           return super().test_attention_slicing_forward_pass()







@nightly
@require_torch_gpu
class 		a_	(							unittest.TestCase					):



      """simple docstring"""







      def 	_lowerCAmelCase					(				self					:							Union[str, Any]    ):
           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()


      def 	_lowerCAmelCase					(				self					:							Tuple    ):
           SCREAMING_SNAKE_CASE										=load_image(
               'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/'
               'repaint/celeba_hq_256.png'    )
           SCREAMING_SNAKE_CASE										=load_image(
               'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/repaint/mask_256.png'    )
           SCREAMING_SNAKE_CASE										=load_numpy(
               'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/'
               'repaint/celeba_hq_256_result.npy'    )

           SCREAMING_SNAKE_CASE										='google/ddpm-ema-celebahq-256'
           SCREAMING_SNAKE_CASE										=UNetaDModel.from_pretrained(snake_case    )
           SCREAMING_SNAKE_CASE										=RePaintScheduler.from_pretrained(snake_case    )

           SCREAMING_SNAKE_CASE										=RePaintPipeline(unet=snake_case      ,scheduler=snake_case    ).to(snake_case    )
           repaint.set_progress_bar_config(disable=snake_case    )
           repaint.enable_attention_slicing()

           SCREAMING_SNAKE_CASE										=torch.manual_seed(0    )
           SCREAMING_SNAKE_CASE										=repaint(
               snake_case      ,snake_case      ,num_inference_steps=250      ,eta=0.0      ,jump_length=10      ,jump_n_sample=10      ,generator=snake_case      ,output_type='np'      ,)
           SCREAMING_SNAKE_CASE										=output.images[0]

           assert image.shape == (256, 256, 3)
           assert np.abs(expected_image - image    ).mean() < 1e-2

