
'''simple docstring'''
import unittest

from transformers.testing_utils import is_torch_available, require_torch, tooslow

from ...generation.test_utils import torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          CpmAntConfig,
          CpmAntForCausalLM,
          CpmAntModel,
          CpmAntTokenizer,
      )






@require_torch
class 				UpperCAmelCase_			:





 def __init__(      self	,    lowercase_	,    lowercase_=2	,    lowercase_=8	,    lowercase_=True	,    lowercase_=False	,    lowercase_=False	,    lowercase_=False	,    lowercase_=False	,    lowercase_=99	,    lowercase_=32	,    lowercase_=3	,    lowercase_=4	,    lowercase_=37	,    lowercase_=32	,    lowercase_=1_28	,    lowercase_=8	,    lowercase_=8	,    lowercase_=8	,    lowercase_=1.0	,    lowercase_=True	,    ):
     snake_case_	:      List[Any]            =  parent
     snake_case_	:      Union[str, Any]            =  batch_size
     snake_case_	:      Optional[Any]            =  seq_length
     snake_case_	:      Tuple            =  is_training
     snake_case_	:      Tuple            =  use_token_type_ids
     snake_case_	:      Dict            =  use_input_mask
     snake_case_	:      List[str]            =  use_labels
     snake_case_	:      Optional[int]            =  use_mc_token_ids
     snake_case_	:      List[Any]            =  vocab_size
     snake_case_	:      Optional[Any]            =  hidden_size
     snake_case_	:      int            =  num_hidden_layers
     snake_case_	:      Optional[Any]            =  num_attention_heads
     snake_case_	:      str            =  intermediate_size
     snake_case_	:      Union[str, Any]            =  num_buckets
     snake_case_	:      Tuple            =  max_distance
     snake_case_	:      str            =  prompt_length
     snake_case_	:      int            =  prompt_types
     snake_case_	:      List[str]            =  segment_types
     snake_case_	:      List[Any]            =  init_std
     snake_case_	:      Optional[Any]            =  return_dict





 def     snake_case__						(      self):
     snake_case_	:      Optional[int]            =  {}
     snake_case_	:      Tuple            =  ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size).type(torch.intaa)
     snake_case_	:      Union[str, Any]            =  False

     snake_case_	:      Dict            =  self.get_config()

     return (config, input_ids)





 def     snake_case__						(      self):
     return CpmAntConfig(
         vocab_size=self.vocab_size	,    hidden_size=self.hidden_size	,    num_hidden_layers=self.num_hidden_layers	,    num_attention_heads=self.num_attention_heads	,    dim_ff=self.intermediate_size	,    position_bias_num_buckets=self.num_buckets	,    position_bias_max_distance=self.max_distance	,    prompt_types=self.prompt_types	,    prompt_length=self.prompt_length	,    segment_types=self.segment_types	,    use_cache=__SCREAMING_SNAKE_CASE	,    init_std=self.init_std	,    return_dict=self.return_dict	,    )





 def     snake_case__						(      self	,    lowercase_	,    lowercase_	,    *lowercase_):
     snake_case_	:      Optional[Any]            =  CpmAntModel(config=__SCREAMING_SNAKE_CASE)
     model.to(__SCREAMING_SNAKE_CASE)
     model.eval()

     snake_case_	:      int            =  model(**__SCREAMING_SNAKE_CASE).last_hidden_state

     self.parent.assertEqual(hidden_states.shape	,    (self.batch_size, self.seq_length, config.hidden_size))





 def     snake_case__						(      self	,    lowercase_	,    lowercase_	,    *lowercase_):
     snake_case_	:      str            =  CpmAntForCausalLM(__SCREAMING_SNAKE_CASE)
     model.to(__SCREAMING_SNAKE_CASE)
     snake_case_	:      Optional[int]            =  input_ids["input_ids"].to(__SCREAMING_SNAKE_CASE)
     model.eval()

     snake_case_	:      Optional[int]            =  model(**__SCREAMING_SNAKE_CASE)
     self.parent.assertEqual(
         model_output.logits.shape	,    (self.batch_size, self.seq_length, config.vocab_size + config.prompt_types * config.prompt_length)	,    )





 def     snake_case__						(      self):
     snake_case_     ,	snake_case_	:      Any            =  self.prepare_config_and_inputs()
     return config, inputs_dict






@require_torch
class 				UpperCAmelCase_			(		UpperCamelCase_      ,  UpperCamelCase_      ,  unittest.TestCase   ):
 UpperCAmelCase_						      =  (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
 UpperCAmelCase_						      =  (
     {"""feature-extraction""": CpmAntModel, """text-generation""": CpmAntForCausalLM} if is_torch_available() else {}
 )

 UpperCAmelCase_						      =  False
 UpperCAmelCase_						      =  False
 UpperCAmelCase_						      =  False
 UpperCAmelCase_						      =  False
 UpperCAmelCase_						      =  False





 def     snake_case__						(      self):
     snake_case_	:      Union[str, Any]            =  CpmAntModelTester(self)
     snake_case_	:      Any            =  ConfigTester(self	,    config_class=__SCREAMING_SNAKE_CASE)





 def     snake_case__						(      self):
     self.config_tester.create_and_test_config_common_properties()
     self.config_tester.create_and_test_config_to_json_string()
     self.config_tester.create_and_test_config_to_json_file()
     self.config_tester.create_and_test_config_from_and_save_pretrained()
     self.config_tester.check_config_can_be_init_without_params()
     self.config_tester.check_config_arguments_init()





 def     snake_case__						(      self):
     unittest.skip("CPMAnt doesn\'t support input_embeds.")(self.test_inputs_embeds)





 def     snake_case__						(      self):
     unittest.skip(
         "CPMAnt doesn\'t support retain grad in hidden_states or attentions, because prompt management will peel off the output.hidden_states from graph.\\n                 So is attentions. We strongly recommand you use loss to tune model.")(self.test_retain_grad_hidden_states_attentions)





 def     snake_case__						(      self):
     snake_case_     ,	snake_case_	:      Dict            =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_cpmant_model(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE)





 def     snake_case__						(      self):
     snake_case_     ,	snake_case_	:      Optional[int]            =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_lm_head_model(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE)






@require_torch
class 				UpperCAmelCase_			(		unittest.TestCase   ):





 @tooslow
 def     snake_case__						(      self):
     snake_case_	:      Union[str, Any]            =  "今天天气真好！"
     snake_case_	:      Any            =  "openbmb/cpm-ant-10b"
     snake_case_	:      Any            =  CpmAntModel.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      Union[str, Any]            =  CpmAntTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      str            =  tokenizer(__SCREAMING_SNAKE_CASE	,    return_tensors="pt")
     snake_case_	:      str            =  model(**__SCREAMING_SNAKE_CASE).last_hidden_state

     snake_case_	:      Dict            =  torch.tensor(
         [[[6.1_708, 5.9_244, 1.0_835], [6.5_207, 6.2_893, -11.33_24], [-1.0_107, -0.0_576, -5.9_577]]]	,    )
     self.assertTrue(torch.allclose(hidden_states[:, :3, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1E-2))






@require_torch
class 				UpperCAmelCase_			(		unittest.TestCase   ):





 @tooslow
 def     snake_case__						(      self):
     snake_case_	:      Optional[Any]            =  "今天天气真好！"
     snake_case_	:      List[str]            =  "openbmb/cpm-ant-10b"
     snake_case_	:      Any            =  CpmAntForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      List[str]            =  CpmAntTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      List[str]            =  tokenizer(__SCREAMING_SNAKE_CASE	,    return_tensors="pt")
     snake_case_	:      Optional[int]            =  model(**__SCREAMING_SNAKE_CASE).logits

     snake_case_	:      Optional[int]            =  torch.tensor(
         [[[-6.4_267, -6.4_083, -6.3_958], [-5.8_802, -5.9_447, -5.7_811], [-5.3_896, -5.4_820, -5.4_295]]]	,    )
     self.assertTrue(torch.allclose(hidden_states[:, :3, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1E-2))





 @tooslow
 def     snake_case__						(      self):
     snake_case_	:      str            =  "openbmb/cpm-ant-10b"
     snake_case_	:      List[str]            =  CpmAntForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      Union[str, Any]            =  CpmAntTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      Any            =  "今天天气不错，"
     snake_case_	:      int            =  "今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的"
     snake_case_	:      Dict            =  tokenizer(__SCREAMING_SNAKE_CASE	,    return_tensors="pt")
     snake_case_	:      int            =  model.generate(**__SCREAMING_SNAKE_CASE)
     snake_case_	:      Optional[int]            =  tokenizer.batch_decode(__SCREAMING_SNAKE_CASE)
     self.assertEqual(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE)





 @tooslow
 def     snake_case__						(      self):
     snake_case_	:      Tuple            =  "openbmb/cpm-ant-10b"
     snake_case_	:      Optional[Any]            =  CpmAntForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      str            =  CpmAntTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
     snake_case_	:      Optional[int]            =  ["今天天气不错，", "新年快乐，万事如意！"]
     snake_case_	:      List[str]            =  [
         "今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的",
         "新年快乐，万事如意！在这辞旧迎新的美好时刻，我谨代表《农村新技术》杂志社全体同仁，向一直以来关心、支持《农村新技术》杂志发展的各级领导、各界朋友和广大读者致以最诚挚的",
     ]
     snake_case_	:      str            =  tokenizer(__SCREAMING_SNAKE_CASE	,    return_tensors="pt"	,    padding=__SCREAMING_SNAKE_CASE)
     snake_case_	:      Dict            =  model.generate(**__SCREAMING_SNAKE_CASE)
     snake_case_	:      Any            =  tokenizer.batch_decode(__SCREAMING_SNAKE_CASE)
     self.assertEqual(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE)






'''simple docstring'''
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_     =  logging.get_logger(__name__)






class 				UpperCAmelCase_			(		snake_case__   ):
 UpperCAmelCase_						      =  ["""input_ids""", """attention_mask"""]





 def __init__(      self	,    lowercase_="[PAD]"	,    lowercase_="[BOS]"	,    lowercase_="[EOS]"	,    lowercase_="[MASK]"	,    lowercase_="[CLS]"	,    lowercase_="[SEP]"	,    lowercase_=20_48	,    **lowercase_	,    ):
     snake_case_	:      int            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else pad_token
     snake_case_	:      List[str]            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else bos_token
     snake_case_	:      Tuple            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else eos_token
     snake_case_	:      Optional[Any]            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else mask_token
     snake_case_	:      List[str]            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else cls_token
     snake_case_	:      int            =  AddedToken(lowercase_	,    lstrip=lowercase_	,    rstrip=lowercase_) if isinstance(lowercase_	,    lowercase_) else sep_token

     super().__init__(
         pad_token=lowercase_	,    bos_token=lowercase_	,    eos_token=lowercase_	,    mask_token=lowercase_	,    cls_token=lowercase_	,    sep_token=lowercase_	,    model_max_length=lowercase_	,    **lowercase_	,    )

     snake_case_	:      Dict            =  2**8  # utf is 8 bits

     # define special tokens dict
     snake_case_	:      Dict[str, int]            =  {
         self.pad_token: 0,
         self.bos_token: 1,
         self.eos_token: 2,
         self.mask_token: 3,
         self.cls_token: 4,
         self.sep_token: 5,
     }
     snake_case_	:      Dict            =  len(self.special_tokens_encoder)
     snake_case_	:      Dict[int, str]            =  {v: k for k, v in self.special_tokens_encoder.items()}





 def     snake_case__						(      self):
     snake_case_	:      str            =  self.special_tokens_encoder.copy()
     vocab.update(self.added_tokens_encoder)
     for i in range(self._utf_vocab_size):
         snake_case_	:      Tuple            =  chr(lowercase_)
         snake_case_	:      Optional[Any]            =  i + len(self.special_tokens_encoder)
     return vocab





 @property
 def     snake_case__						(      self):
     return self._utf_vocab_size + self._num_special_tokens





 def     snake_case__						(      self	,    lowercase_	,    lowercase_ = None	,    lowercase_ = False):
     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowercase_	,    token_ids_a=lowercase_	,    already_has_special_tokens=lowercase_)

     # normal case: some special tokens
     if token_ids_a is None:
         return [1] + [0] * len(lowercase_) + [1]
     return [1] + ([0] * len(lowercase_)) + [1] + ([0] * len(lowercase_)) + [1]





 def     snake_case__						(      self	,    lowercase_	,    lowercase_ = None):
     if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     else:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]





 def     snake_case__						(      self	,    lowercase_):
     snake_case_	:      Tuple            =  [chr(lowercase_) for i in text.encode("utf-8")]
     return tokens





 def     snake_case__						(      self	,    lowercase_):
     if token in self.special_tokens_encoder:
         snake_case_	:      Optional[int]            =  self.special_tokens_encoder[token]
     elif token in self.added_tokens_encoder:
         snake_case_	:      str            =  self.added_tokens_encoder[token]
     elif len(lowercase_) != 1:
         snake_case_	:      Union[str, Any]            =  self.unk_token_id
     else:
         snake_case_	:      Optional[Any]            =  ord(lowercase_) + self._num_special_tokens
     return token_id





 def     snake_case__						(      self	,    lowercase_):
     if index in self.special_tokens_decoder:
         snake_case_	:      Optional[Any]            =  self.special_tokens_decoder[index]
     elif index in self.added_tokens_decoder:
         snake_case_	:      List[str]            =  self.added_tokens_decoder[index]
     else:
         snake_case_	:      Union[str, Any]            =  chr(index - self._num_special_tokens)
     return token





 def     snake_case__						(      self	,    lowercase_):
     snake_case_	:      str            =  b""
     for token in tokens:
         if token in self.special_tokens_decoder:
             snake_case_	:      List[Any]            =  self.special_tokens_decoder[token].encode("utf-8")
         elif token in self.added_tokens_decoder:
             snake_case_	:      Any            =  self.special_tokens_decoder[token].encode("utf-8")
         elif token in self.special_tokens_encoder:
             snake_case_	:      Union[str, Any]            =  token.encode("utf-8")
         elif token in self.added_tokens_encoder:
             snake_case_	:      Any            =  token.encode("utf-8")
         else:
             snake_case_	:      Union[str, Any]            =  bytes([ord(lowercase_)])
         bstring += tok_string
     snake_case_	:      Union[str, Any]            =  bstring.decode("utf-8"	,    errors="replace")
     return string





 def     snake_case__						(      self	,    lowercase_	,    lowercase_ = None):
     return ()




