




import copy
import logging
import random

import joblib
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from tqdm import tqdm

from transformers import AdamW, GPTaLMHeadModel, get_linear_schedule_with_warmup


a_										=       logging.getLogger(__name__)




def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		List[Any]:

    """simple docstring"""


    random.seed(lowercase_ )
    np.random.seed(lowercase_ )
    torch.manual_seed(lowercase_ )
    torch.cuda.manual_seed_all(lowercase_ )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_ )    ->		List[str]:

    """simple docstring"""


    model.eval()
    __UpperCamelCase											=  next(model.parameters() ).device
    __UpperCamelCase											=  1
    __UpperCamelCase											=  torch.zeros((eval_batch_size, context_len)   ,       dtype=torch.long   ,       device=lowercase_ )
    __UpperCamelCase											=  DataLoader(lowercase_   ,       shuffle=lowercase_   ,       batch_size=lowercase_ )
    __UpperCamelCase											=  torch.zeros(1   ,       device=lowercase_ )
    __UpperCamelCase											=  0
    for batch in eval_dataloader:
        batch.to(lowercase_ )
        # pad
        context.zero_()
        for i in range(lowercase_ ):
            __UpperCamelCase											=  batch[i]
        __UpperCamelCase											=  model(lowercase_   ,       labels=lowercase_ )
        eval_loss += outputs[0].sum().item()
        nb_eval_examples += batch.size(0 )
    __UpperCamelCase											=  eval_loss / nb_eval_examples
    __UpperCamelCase											=  torch.exp(lowercase_ )
    model.train()
    return perplexity




def    __SCREAMING_SNAKE_CASE				(					lowercase_="gpt2" )    ->		Optional[Any]:

    """simple docstring"""


    __UpperCamelCase											=  GPTaLMHeadModel.from_pretrained(lowercase_   ,       output_hidden_states=lowercase_ )
    torch.save(model.state_dict()   ,       model_name + '''local.pt''' )
    return model




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_ )    ->		Optional[Any]:

    """simple docstring"""


    __UpperCamelCase											=  copy.deepcopy(lowercase_ )
    model.to(lowercase_ )

    __UpperCamelCase											=  ['''bias''', '''LayerNorm.weight''']
    __UpperCamelCase											=  [
        {
            '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay )],
            '''weight_decay''': 0.0,
        },
        {'''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay )], '''weight_decay''': 0.0},
    ]
    __UpperCamelCase											=  AdamW(lowercase_   ,       lr=5E-5   ,       eps=1E-8 )
    __UpperCamelCase											=  get_linear_schedule_with_warmup(lowercase_   ,       0   ,       lowercase_ )
    torch.cuda.empty_cache()
    return model, lm_optimizer, lm_scheduler




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_ )    ->		Optional[int]:

    """simple docstring"""


    __UpperCamelCase											=  np.array(lowercase_ ).mean()
    __UpperCamelCase											=  np.array(lowercase_ ).std()
    __UpperCamelCase											=  (real_perps - avg) / std
    __UpperCamelCase											=  list(zip(lowercase_   ,       lowercase_   ,       lowercase_ ) )
    joblib.dump(lowercase_   ,       lowercase_ )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_="dev.jbl"   ,       lowercase_=recopy_gpta   ,       )    ->		str:

    """simple docstring"""


    __UpperCamelCase											=  []
    __UpperCamelCase											=  []
    __UpperCamelCase											=  []

    # Initialize the transformer model
    __UpperCamelCase											=  copy.deepcopy(lowercase_ )
    orig_model.to(device='''cpu''' )
    torch.cuda.empty_cache()

    # Compute perplexity of initial transformer model for comparison
    model.train()
    __UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase											=  recopy_model(lowercase_   ,       lowercase_   ,       lowercase_ )

    for step in tqdm(range(lowercase_ ) ):
        __UpperCamelCase											=  torch.zeros((1, context_len)   ,       dtype=torch.long   ,       device=lowercase_ )
        __UpperCamelCase											=  random.choice(lowercase_ )
        __UpperCamelCase											=  random.randint(0   ,       len(story[0] ) - context_len - 1 )
        __UpperCamelCase											=  story[0][start : start + context_len]
        lm_optimizer.zero_grad()
        __UpperCamelCase											=  model(lowercase_   ,       labels=lowercase_ )
        __UpperCamelCase											=  outputs[0]
        __UpperCamelCase											=  compute_perplexity(lowercase_   ,       lowercase_   ,       lowercase_ )
        model.train()
        lm_loss.backward()
        # Do LM backprop
        torch.nn.utils.clip_grad_norm_(model.parameters()   ,       3.0 )
        lm_optimizer.step()
        lm_scheduler.step()  # Update learning rate schedule

        # Compute perplexity after back-propagating on the selected context
        __UpperCamelCase											=  compute_perplexity(lowercase_   ,       lowercase_   ,       lowercase_ )

        # Periodically save the stored (X, IG(X)) pairs
        if step % 10_00 == 0 and step > 1:
            intermittent_save(lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_ )

        # Reset the pretrained model to the original pretrained GPT-2 weights after each iteration
        __UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase											=  recopy_model(lowercase_   ,       lowercase_   ,       lowercase_ )

        past_perps.append(past_perp.item() )
        real_perps.append(orig_perp - real_perp.item() )
        contexts.append(np.array(context.cpu() ) )

    intermittent_save(lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_ )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_="data/tokenized_stories_train_wikitext103.jbl"   ,       lowercase_=1_00   ,       lowercase_=10_26   ,       lowercase_=True )    ->		int:

    """simple docstring"""


    __UpperCamelCase											=  joblib.load(lowercase_ )
    print('''data loaded''' )
    __UpperCamelCase											=  []
    if trim:
        for i, example in enumerate(lowercase_ ):
            if len(example[0] ) > min_len:
                __UpperCamelCase											=  random.randint(0   ,       len(example[0] ) - context_len - 1 )
                objective_set.append(example[0, start : start + context_len] )
            if len(lowercase_ ) >= number:
                break
        __UpperCamelCase											=  []
        for j in range(i + 1   ,       len(lowercase_ ) ):
            if len(data[j][0] ) > min_len:
                train_data.append(data[j] )
    else:
        __UpperCamelCase											=  data[0:number]
        __UpperCamelCase											=  data[number:]

    joblib.dump(lowercase_   ,       '''objective_set.jbl''' )
    print('''objective set saved''' )
    return train_data, objective_set




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_=50   ,       lowercase_="secondary_learner.pt" )    ->		Any:

    """simple docstring"""


    __UpperCamelCase											=  torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu''' )
    # We will use the first 512 pairs from our dataset as a test set for
    # our secondary learner and the rest to train
    __UpperCamelCase											=  train_dataset[:5_12]
    __UpperCamelCase											=  train_dataset[5_12:]
    __UpperCamelCase											=  DataLoader(lowercase_   ,       shuffle=lowercase_   ,       batch_size=lowercase_ )
    __UpperCamelCase											=  DataLoader(lowercase_   ,       shuffle=lowercase_   ,       batch_size=lowercase_ )

    # secondary learner model set up
    __UpperCamelCase											=  nn.MSELoss()
    __UpperCamelCase											=  nn.MSELoss(reduction='''sum''' )
    secondary_learner.to(lowercase_ )
    __UpperCamelCase											=  torch.optim.Adam(secondary_learner.parameters()   ,       lr=0.0_0001 )
    secondary_learner.train()

    # TODO in original code this is written as number of actual batches seen
    # not number of items seen but other places it is number of items instead.
    # improve consistency! changed this to epochs for clarity
    __UpperCamelCase											=  float('''inf''' )
    # Iterate through batches until we've used max_steps batches
    for epoch in range(int(lowercase_ ) ):
        __UpperCamelCase											=  0.0
        secondary_learner.train()
        for step, batch in enumerate(lowercase_ ):
            __UpperCamelCase											=  batch[0].to(lowercase_ )
            __UpperCamelCase											=  batch[1].to(lowercase_ )
            __UpperCamelCase											=  secondary_learner(lowercase_ )
            q_optimizer.zero_grad()
            __UpperCamelCase											=  loss(lowercase_   ,       real_q.float() )
            q_loss.backward()
            q_optimizer.step()
            tr_q_loss += q_loss.item()

            # model trains fairly quickly so we won't wait for a full epoch
            # eval is triggered at eval_freq and end of epochs
            if (step % eval_freq == 0 and step > 0) or ((step + 1) == len(lowercase_ )):
                __UpperCamelCase											=  tr_q_loss / (step + 1)

                secondary_learner.eval()
                __UpperCamelCase											=  0.0
                __UpperCamelCase											=  0.0
                __UpperCamelCase											=  []
                __UpperCamelCase											=  []
                # Compute performance of the secondary learner after this batch
                for stepa, batcha in enumerate(lowercase_ ):
                    __UpperCamelCase											=  batcha[0].to(lowercase_ )
                    __UpperCamelCase											=  batcha[1].to(lowercase_ )
                    __UpperCamelCase											=  secondary_learner(lowercase_ )
                    q_lossa += test_loss(lowercase_   ,       lowercase_ ).item()
                    sum_qa += torch.sum(lowercase_ ).item()
                    for ei, i in enumerate(predicted_qa.cpu().detach().numpy() ):
                        predicted.append(i.item() )
                    for ei, i in enumerate(real_qa.cpu().detach().numpy() ):
                        actual.append(i.item() )

                q_lossa /= len(lowercase_ )
                print(
                    '''Epoch: '''   ,       lowercase_   ,       '''step: '''   ,       lowercase_   ,       '''Avg. q:'''   ,       sum_qa / len(lowercase_ )   ,       '''Train Loss: '''   ,       lowercase_   ,       '''Test Loss: '''   ,       lowercase_   ,       )
                if q_lossa < best_test_loss:
                    joblib.dump((predicted, actual)   ,       '''pred_vs_actual.jbl''' )
                    torch.save(secondary_learner.state_dict()   ,       lowercase_ )
                    __UpperCamelCase											=  q_lossa

            secondary_learner.train()
    return secondary_learner


class 	_lowerCamelCase  (     nn.Module				):

      """simple docstring"""

      def __init__(       self   :	Optional[int]	,    snake_case   :	int			):
          super(snake_case	,    self			).__init__()
          __UpperCamelCase											=  model.transformer.wte
          __UpperCamelCase											=  copy.deepcopy(model.transformer.wte.weight			)
          __UpperCamelCase											=  nn.Convad(self.embeddings.weight.size(1			)	,    256	,    3	,    padding=1			)
          __UpperCamelCase											=  nn.Sequential(nn.Linear(256	,    32			)	,    nn.Dropout(p=0.1			)	,    nn.Linear(32	,    32			)	,    nn.Linear(32	,    1			)			)







      def  snake_case		(       self   :	Any	,    snake_case   :	Optional[Any]			):
          __UpperCamelCase											=  torch.max(self.conv(self.embeddings(snake_case			).squeeze(1			).transpose(1	,    2			)			)	,    2			)[0]
          __UpperCamelCase											=  self.fc(snake_case			)
          return qs.squeeze(1			)







      @classmethod
      def  snake_case		(       cls   :	Union[str, Any]	,    snake_case   :	List[Any]	,    snake_case   :	Tuple			):
          __UpperCamelCase											=  cls(snake_case			)  # this calls __init__
          __UpperCamelCase											=  torch.load(snake_case			)
          secondary_learner.load_state_dict(snake_case			)
          __UpperCamelCase											=  model.transformer.wte
          __UpperCamelCase											=  copy.deepcopy(model.transformer.wte.weight			)
          return secondary_learner












import unittest

from transformers.testing_utils import is_torch_available, require_torch, tooslow

from ...generation.test_utils import torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     CpmAntConfig,
     CpmAntForCausalLM,
     CpmAntModel,
     CpmAntTokenizer,
 )

@require_torch
class 	_lowerCamelCase  :

      """simple docstring"""

      def __init__(       self   :	str	,    snake_case   :	str	,    snake_case   :	List[str]=2	,    snake_case   :	Optional[Any]=8	,    snake_case   :	List[Any]=True	,    snake_case   :	Optional[Any]=False	,    snake_case   :	str=False	,    snake_case   :	Tuple=False	,    snake_case   :	str=False	,    snake_case   :	int=99	,    snake_case   :	Tuple=32	,    snake_case   :	Union[str, Any]=3	,    snake_case   :	str=4	,    snake_case   :	List[Any]=37	,    snake_case   :	Tuple=32	,    snake_case   :	Tuple=128	,    snake_case   :	Optional[int]=8	,    snake_case   :	Optional[Any]=8	,    snake_case   :	List[Any]=8	,    snake_case   :	List[str]=1.0	,    snake_case   :	List[Any]=True	,    ):
          __UpperCamelCase											=  parent
          __UpperCamelCase											=  batch_size
          __UpperCamelCase											=  seq_length
          __UpperCamelCase											=  is_training
          __UpperCamelCase											=  use_token_type_ids
          __UpperCamelCase											=  use_input_mask
          __UpperCamelCase											=  use_labels
          __UpperCamelCase											=  use_mc_token_ids
          __UpperCamelCase											=  vocab_size
          __UpperCamelCase											=  hidden_size
          __UpperCamelCase											=  num_hidden_layers
          __UpperCamelCase											=  num_attention_heads
          __UpperCamelCase											=  intermediate_size
          __UpperCamelCase											=  num_buckets
          __UpperCamelCase											=  max_distance
          __UpperCamelCase											=  prompt_length
          __UpperCamelCase											=  prompt_types
          __UpperCamelCase											=  segment_types
          __UpperCamelCase											=  init_std
          __UpperCamelCase											=  return_dict







      def  snake_case		(       self   :	Tuple			):
          __UpperCamelCase											=  {}
          __UpperCamelCase											=  ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size			).type(torch.intaa			)
          __UpperCamelCase											=  False

          __UpperCamelCase											=  self.get_config()

          return (config, input_ids)







      def  snake_case		(       self   :	List[str]			):
          return CpmAntConfig(
              vocab_size=self.vocab_size	,    hidden_size=self.hidden_size	,    num_hidden_layers=self.num_hidden_layers	,    num_attention_heads=self.num_attention_heads	,    dim_ff=self.intermediate_size	,    position_bias_num_buckets=self.num_buckets	,    position_bias_max_distance=self.max_distance	,    prompt_types=self.prompt_types	,    prompt_length=self.prompt_length	,    segment_types=self.segment_types	,    use_cache=snake_case	,    init_std=self.init_std	,    return_dict=self.return_dict	,    )







      def  snake_case		(       self   :	Union[str, Any]	,    snake_case   :	Tuple	,    snake_case   :	List[str]	,    *snake_case   :	str			):
          __UpperCamelCase											=  CpmAntModel(config=snake_case			)
          model.to(snake_case			)
          model.eval()

          __UpperCamelCase											=  model(**snake_case			).last_hidden_state

          self.parent.assertEqual(hidden_states.shape	,    (self.batch_size, self.seq_length, config.hidden_size)			)







      def  snake_case		(       self   :	List[str]	,    snake_case   :	Optional[int]	,    snake_case   :	List[str]	,    *snake_case   :	Any			):
          __UpperCamelCase											=  CpmAntForCausalLM(snake_case			)
          model.to(snake_case			)
          __UpperCamelCase											=  input_ids['''input_ids'''].to(snake_case			)
          model.eval()

          __UpperCamelCase											=  model(**snake_case			)
          self.parent.assertEqual(
              model_output.logits.shape	,    (self.batch_size, self.seq_length, config.vocab_size + config.prompt_types * config.prompt_length)	,    )







      def  snake_case		(       self   :	List[str]			):
          __UpperCamelCase						, __UpperCamelCase											=  self.prepare_config_and_inputs()
          return config, inputs_dict







@require_torch
class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE							,     unittest.TestCase				):

      """simple docstring"""

      lowerCAmelCase__     :	List[Any]      =   (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
      lowerCAmelCase__     :	Union[str, Any]      =   (
          {"feature-extraction": CpmAntModel, "text-generation": CpmAntForCausalLM} if is_torch_available() else {}
      )

      lowerCAmelCase__     :	Optional[int]      =   False
      lowerCAmelCase__     :	Union[str, Any]      =   False
      lowerCAmelCase__     :	Optional[int]      =   False
      lowerCAmelCase__     :	Tuple      =   False
      lowerCAmelCase__     :	Union[str, Any]      =   False







      def  snake_case		(       self   :	Any			):
          __UpperCamelCase											=  CpmAntModelTester(self			)
          __UpperCamelCase											=  ConfigTester(self	,    config_class=snake_case			)







      def  snake_case		(       self   :	Optional[Any]			):
          self.config_tester.create_and_test_config_common_properties()
          self.config_tester.create_and_test_config_to_json_string()
          self.config_tester.create_and_test_config_to_json_file()
          self.config_tester.create_and_test_config_from_and_save_pretrained()
          self.config_tester.check_config_can_be_init_without_params()
          self.config_tester.check_config_arguments_init()







      def  snake_case		(       self   :	List[str]			):
          unittest.skip('''CPMAnt doesn\'t support input_embeds.'''			)(self.test_inputs_embeds			)







      def  snake_case		(       self   :	int			):
          unittest.skip(
              '''CPMAnt doesn\'t support retain grad in hidden_states or attentions, because prompt management will peel off the output.hidden_states from graph.\
                 So is attentions. We strongly recommand you use loss to tune model.'''			)(self.test_retain_grad_hidden_states_attentions			)







      def  snake_case		(       self   :	Union[str, Any]			):
          __UpperCamelCase						, __UpperCamelCase											=  self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_cpmant_model(snake_case	,    snake_case			)







      def  snake_case		(       self   :	Any			):
          __UpperCamelCase						, __UpperCamelCase											=  self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head_model(snake_case	,    snake_case			)







@require_torch
class 	_lowerCamelCase  (     unittest.TestCase				):

      """simple docstring"""

      @tooslow
      def  snake_case		(       self   :	Dict			):
          __UpperCamelCase											=  '''今天天气真好！'''
          __UpperCamelCase											=  '''openbmb/cpm-ant-10b'''
          __UpperCamelCase											=  CpmAntModel.from_pretrained(snake_case			)
          __UpperCamelCase											=  CpmAntTokenizer.from_pretrained(snake_case			)
          __UpperCamelCase											=  tokenizer(snake_case	,    return_tensors='''pt'''			)
          __UpperCamelCase											=  model(**snake_case			).last_hidden_state

          __UpperCamelCase											=  torch.tensor(
              [[[6.17_08, 5.92_44, 1.08_35], [6.52_07, 6.28_93, -11.33_24], [-1.01_07, -0.05_76, -5.95_77]]]	,    )
          self.assertTrue(torch.allclose(hidden_states[:, :3, :3]	,    snake_case	,    atol=1E-2			)			)







@require_torch
class 	_lowerCamelCase  (     unittest.TestCase				):

      """simple docstring"""

      @tooslow
      def  snake_case		(       self   :	Optional[int]			):
          __UpperCamelCase											=  '''今天天气真好！'''
          __UpperCamelCase											=  '''openbmb/cpm-ant-10b'''
          __UpperCamelCase											=  CpmAntForCausalLM.from_pretrained(snake_case			)
          __UpperCamelCase											=  CpmAntTokenizer.from_pretrained(snake_case			)
          __UpperCamelCase											=  tokenizer(snake_case	,    return_tensors='''pt'''			)
          __UpperCamelCase											=  model(**snake_case			).logits

          __UpperCamelCase											=  torch.tensor(
              [[[-6.42_67, -6.40_83, -6.39_58], [-5.88_02, -5.94_47, -5.78_11], [-5.38_96, -5.48_20, -5.42_95]]]	,    )
          self.assertTrue(torch.allclose(hidden_states[:, :3, :3]	,    snake_case	,    atol=1E-2			)			)







      @tooslow
      def  snake_case		(       self   :	List[str]			):
          __UpperCamelCase											=  '''openbmb/cpm-ant-10b'''
          __UpperCamelCase											=  CpmAntForCausalLM.from_pretrained(snake_case			)
          __UpperCamelCase											=  CpmAntTokenizer.from_pretrained(snake_case			)
          __UpperCamelCase											=  '''今天天气不错，'''
          __UpperCamelCase											=  '''今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的'''
          __UpperCamelCase											=  tokenizer(snake_case	,    return_tensors='''pt'''			)
          __UpperCamelCase											=  model.generate(**snake_case			)
          __UpperCamelCase											=  tokenizer.batch_decode(snake_case			)
          self.assertEqual(snake_case	,    snake_case			)







      @tooslow
      def  snake_case		(       self   :	Any			):
          __UpperCamelCase											=  '''openbmb/cpm-ant-10b'''
          __UpperCamelCase											=  CpmAntForCausalLM.from_pretrained(snake_case			)
          __UpperCamelCase											=  CpmAntTokenizer.from_pretrained(snake_case			)
          __UpperCamelCase											=  ['''今天天气不错，''', '''新年快乐，万事如意！''']
          __UpperCamelCase											=  [
              '''今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的''',
              '''新年快乐，万事如意！在这辞旧迎新的美好时刻，我谨代表《农村新技术》杂志社全体同仁，向一直以来关心、支持《农村新技术》杂志发展的各级领导、各界朋友和广大读者致以最诚挚的''',
          ]
          __UpperCamelCase											=  tokenizer(snake_case	,    return_tensors='''pt'''	,    padding=snake_case			)
          __UpperCamelCase											=  model.generate(**snake_case			)
          __UpperCamelCase											=  tokenizer.batch_decode(snake_case			)
          self.assertEqual(snake_case	,    snake_case			)






