



'''simple docstring'''





import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      ErnieMForInformationExtraction,
      ErnieMForMultipleChoice,
      ErnieMForQuestionAnswering,
      ErnieMForSequenceClassification,
      ErnieMForTokenClassification,
      ErnieMModel,
  )
  from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST






class __A      :



 '''simple docstring'''







 def __init__(self			,       A			,       A=13			,       A=7			,       A=True			,       A=True			,       A=True			,       A=99			,       A=32			,       A=5			,       A=4			,       A=37			,       A="gelu"			,       A=0.1			,       A=0.1			,       A=512			,       A=16			,       A=2			,       A=0.02			,       A=3			,       A=4			,       A=None			,       )   ->					Tuple:


      """simple docstring"""



      _a     =  parent
      _a     =  batch_size
      _a     =  seq_length
      _a     =  is_training
      _a     =  use_input_mask
      _a     =  use_labels
      _a     =  vocab_size
      _a     =  hidden_size
      _a     =  num_hidden_layers
      _a     =  num_attention_heads
      _a     =  intermediate_size
      _a     =  hidden_act
      _a     =  hidden_dropout_prob
      _a     =  attention_probs_dropout_prob
      _a     =  max_position_embeddings
      _a     =  type_vocab_size
      _a     =  type_sequence_label_size
      _a     =  initializer_range
      _a     =  num_labels
      _a     =  num_choices
      _a     =  scope



 def  a__  (self	)   ->					str:


      """simple docstring"""



      _a     =  ids_tensor([self.batch_size, self.seq_length]			,       self.vocab_size	)

      _a     =  None
      if self.use_input_mask:
           _a     =  random_attention_mask([self.batch_size, self.seq_length]	)

      _a     =  None
      _a     =  None
      _a     =  None
      if self.use_labels:
           _a     =  ids_tensor([self.batch_size]			,       self.type_sequence_label_size	)
           _a     =  ids_tensor([self.batch_size, self.seq_length]			,       self.num_labels	)
           _a     =  ids_tensor([self.batch_size]			,       self.num_choices	)

      _a     =  self.get_config()

      return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      _a     =  ids_tensor([self.batch_size, self.seq_length]			,       self.vocab_size	)

      _a     =  None
      if self.use_input_mask:
           _a     =  random_attention_mask([self.batch_size, self.seq_length]	)
      _a     =  self.get_config()

      return config, input_ids, input_mask



 def  a__  (self	)   ->					Optional[Any]:


      """simple docstring"""



      return ErnieMConfig(
          vocab_size=self.vocab_size			,       hidden_size=self.hidden_size			,       num_hidden_layers=self.num_hidden_layers			,       num_attention_heads=self.num_attention_heads			,       intermediate_size=self.intermediate_size			,       hidden_act=self.hidden_act			,       hidden_dropout_prob=self.hidden_dropout_prob			,       attention_probs_dropout_prob=self.attention_probs_dropout_prob			,       max_position_embeddings=self.max_position_embeddings			,       type_vocab_size=self.type_vocab_size			,       initializer_range=self.initializer_range			,       )



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					str:


      """simple docstring"""



      _a     =  ErnieMModel(config=A	)
      model.to(A	)
      model.eval()
      _a     =  model(A			,       return_dict=A	)
      self.parent.assertEqual(result.last_hidden_state.shape			,       (self.batch_size, self.seq_length, self.hidden_size)	)



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					Any:


      """simple docstring"""



      _a     =  ErnieMForQuestionAnswering(config=A	)
      model.to(A	)
      model.eval()
      _a     =  model(
          A			,       attention_mask=A			,       start_positions=A			,       end_positions=A			,       )
      self.parent.assertEqual(result.start_logits.shape			,       (self.batch_size, self.seq_length)	)
      self.parent.assertEqual(result.end_logits.shape			,       (self.batch_size, self.seq_length)	)



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					Any:


      """simple docstring"""



      _a     =  ErnieMForInformationExtraction(config=A	)
      model.to(A	)
      model.eval()
      _a     =  torch.ones_like(A			,       dtype=torch.floataa	)
      _a     =  model(
          A			,       attention_mask=A			,       start_positions=A			,       end_positions=A			,       )
      self.parent.assertEqual(result.start_logits.shape			,       (self.batch_size, self.seq_length)	)
      self.parent.assertEqual(result.end_logits.shape			,       (self.batch_size, self.seq_length)	)



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					Tuple:


      """simple docstring"""



      _a     =  self.num_labels
      _a     =  ErnieMForSequenceClassification(A	)
      model.to(A	)
      model.eval()
      _a     =  model(A			,       attention_mask=A			,       labels=A	)
      self.parent.assertEqual(result.logits.shape			,       (self.batch_size, self.num_labels)	)



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					List[str]:


      """simple docstring"""



      _a     =  self.num_labels
      _a     =  ErnieMForTokenClassification(config=A	)
      model.to(A	)
      model.eval()
      input_ids.to(A	)
      input_mask.to(A	)
      token_labels.to(A	)

      _a     =  model(A			,       attention_mask=A			,       labels=A	)

      self.parent.assertEqual(result.logits.shape			,       (self.batch_size, self.seq_length, self.num_labels)	)



 def  a__  (self			,       A			,       A			,       A			,       A			,       A			,       A	)   ->					Optional[int]:


      """simple docstring"""



      _a     =  self.num_choices
      _a     =  ErnieMForMultipleChoice(config=A	)
      model.to(A	)
      model.eval()
      _a     =  input_ids.unsqueeze(1	).expand(-1			,       self.num_choices			,       -1	).contiguous()
      _a     =  input_mask.unsqueeze(1	).expand(-1			,       self.num_choices			,       -1	).contiguous()
      _a     =  model(
          A			,       attention_mask=A			,       labels=A			,       )
      self.parent.assertEqual(result.logits.shape			,       (self.batch_size, self.num_choices)	)



 def  a__  (self	)   ->					str:


      """simple docstring"""



      _a     =  self.prepare_config_and_inputs()
      (
          (
          _a
      )  ,   (
          _a
      )  ,   (
          _a
      )  ,   (
          _a
      )  ,   (
          _a
      )  ,   (
          _a
      )  ,   
      )     =  config_and_inputs
      _a     =  {'''input_ids''': input_ids, '''attention_mask''': input_mask}
      return config, inputs_dict






@require_torch
class __A      (  A       ,       A       ,       unittest.TestCase						):



 '''simple docstring'''







 __lowerCamelCase   :	Union[str, Any]										=	(
     (
         ErnieMModel,
         ErnieMForMultipleChoice,
         ErnieMForQuestionAnswering,
         ErnieMForSequenceClassification,
         ErnieMForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 __lowerCamelCase   :	Optional[Any]										=	()
 __lowerCamelCase   :	Union[str, Any]										=	(
     {
         'feature-extraction': ErnieMModel,
         'question-answering': ErnieMForQuestionAnswering,
         'text-classification': ErnieMForSequenceClassification,
         'token-classification': ErnieMForTokenClassification,
         'zero-shot': ErnieMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowerCamelCase   :	Optional[Any]										=	False



 def  a__  (self			,       A			,       A			,       A			,       A			,       A	)   ->					Tuple:


      """simple docstring"""



      if pipeline_test_casse_name == "QAPipelineTests":
           return True

      return False



 def  a__  (self	)   ->					int:


      """simple docstring"""



      _a     =  ErnieMModelTester(self	)
      _a     =  ConfigTester(self			,       config_class=A			,       hidden_size=37	)



 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      self.config_tester.run_common_tests()



 def  a__  (self	)   ->					int:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*A	)



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
           _a     =  type
           self.model_tester.create_and_check_model(*A	)



 def  a__  (self	)   ->					List[str]:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*A	)



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*A	)



 def  a__  (self	)   ->					List[str]:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*A	)



 def  a__  (self	)   ->					Any:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_information_extraction(*A	)



 def  a__  (self	)   ->					Optional[int]:


      """simple docstring"""



      _a     =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*A	)



 @slow
 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           _a     =  ErnieMModel.from_pretrained(A	)
           self.assertIsNotNone(A	)






@require_torch
class __A      (  unittest.TestCase						):



 '''simple docstring'''







 @slow
 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch'''	)
      model.eval()
      _a     =  torch.tensor([[0, 1, 2, 3, 4, 5]]	)
      _a     =  model(A	)[0]

      # TODO Replace vocab size
      _a     =  768

      _a     =  torch.Size((1, 6, hidden_size)	)
      self.assertEqual(output.shape			,       A	)

      _a     =  torch.tensor(
          [[[-0.0012, 0.1245, -0.0214], [-0.0742, 0.0244, -0.0771], [-0.0333, 0.1164, -0.1554]]]	)

      self.assertTrue(torch.allclose(output[:, :3, :3]			,       A			,       atol=1E-3	)	)








'''simple docstring'''





import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
lowercase_			=  logging.getLogger(__name__)




def lowerCAmelCase  ():



     """simple docstring"""



     _a     =  argparse.ArgumentParser(description='''PyTorch Transformer Language Model''')
     parser.add_argument('''--model_name'''  ,     type=__A  ,     default='''transfo-xl-wt103'''  ,     help='''pretrained model name''')
     parser.add_argument(
         '''--split'''  ,     type=__A  ,     default='''test'''  ,     choices=['''all''', '''valid''', '''test''']  ,     help='''which split to evaluate''')
     parser.add_argument('''--batch_size'''  ,     type=__A  ,     default=10  ,     help='''batch size''')
     parser.add_argument('''--tgt_len'''  ,     type=__A  ,     default=128  ,     help='''number of tokens to predict''')
     parser.add_argument('''--ext_len'''  ,     type=__A  ,     default=0  ,     help='''length of the extended context''')
     parser.add_argument('''--mem_len'''  ,     type=__A  ,     default=1_600  ,     help='''length of the retained previous heads''')
     parser.add_argument('''--clamp_len'''  ,     type=__A  ,     default=1_000  ,     help='''max positional embedding index''')
     parser.add_argument('''--no_cuda'''  ,     action='''store_true'''  ,     help='''Do not use CUDA even though CUA is available''')
     parser.add_argument('''--work_dir'''  ,     type=__A  ,     required=__A  ,     help='''path to the work_dir''')
     parser.add_argument('''--no_log'''  ,     action='''store_true'''  ,     help='''do not log the eval result''')
     parser.add_argument('''--same_length'''  ,     action='''store_true'''  ,     help='''set same length attention with masking''')
     parser.add_argument('''--server_ip'''  ,     type=__A  ,     default=''''''  ,     help='''Can be used for distant debugging.''')
     parser.add_argument('''--server_port'''  ,     type=__A  ,     default=''''''  ,     help='''Can be used for distant debugging.''')
     _a     =  parser.parse_args()
     assert args.ext_len >= 0, "extended context length must be non-negative"

     if args.server_ip and args.server_port:
          # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
          import ptvsd

          print('''Waiting for debugger attach''')
          ptvsd.enable_attach(address=(args.server_ip, args.server_port)  ,     redirect_output=__A)
          ptvsd.wait_for_attach()

     _a     =  torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu''')
     logger.info('''device: {}'''.format(__A))

     # Load a pre-processed dataset
     # You can also build the corpus yourself using TransfoXLCorpus methods
     # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
     # and tokenizing the dataset
     # The pre-processed corpus is a convertion (using the conversion script )
     _a     =  TransfoXLCorpus.from_pretrained(args.model_name)

     _a     =  corpus.get_iterator('''valid'''  ,     args.batch_size  ,     args.tgt_len  ,     device=__A  ,     ext_len=args.ext_len)
     _a     =  corpus.get_iterator('''test'''  ,     args.batch_size  ,     args.tgt_len  ,     device=__A  ,     ext_len=args.ext_len)

     # Load a pre-trained model
     _a     =  TransfoXLLMHeadModel.from_pretrained(args.model_name)
     model.to(__A)

     logger.info(
         '''Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'''.format(
             args.batch_size  ,     args.tgt_len  ,     args.ext_len  ,     args.mem_len  ,     args.clamp_len))

     model.reset_memory_length(args.mem_len)
     if args.clamp_len > 0:
          _a     =  args.clamp_len
     if args.same_length:
          _a     =  True

     ###############################################################################
     # Evaluation code
     ###############################################################################
     def evaluate(__A):
          # Turn on evaluation mode which disables dropout.
          model.eval()
          _a  ,   _a     =  0, 0.0
          _a     =  time.time()
          with torch.no_grad():
               _a     =  None
               for idx, (data, target, seq_len) in enumerate(__A):
                    _a     =  model(__A  ,     lm_labels=__A  ,     mems=__A)
                    _a  ,   _a  ,   _a     =  ret
                    _a     =  loss.mean()
                    total_loss += seq_len * loss.item()
                    total_len += seq_len
               _a     =  time.time() - start_time
          logger.info('''Time : {:.2f}s, {:.2f}ms/segment'''.format(__A  ,     1_000 * total_time / (idx + 1)))
          return total_loss / total_len

     # Run on test data.
     if args.split == "all":
          _a     =  evaluate(__A)
          _a     =  evaluate(__A)
     elif args.split == "valid":
          _a     =  evaluate(__A)
          _a     =  None
     elif args.split == "test":
          _a     =  evaluate(__A)
          _a     =  None

     def format_log(__A  ,     __A):
          _a     =  '''| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '''.format(__A  ,     __A  ,     math.exp(__A))
          return log_str

     _a     =  ''''''
     if valid_loss is not None:
          log_str += format_log(__A  ,     '''valid''')
     if test_loss is not None:
          log_str += format_log(__A  ,     '''test''')

     logger.info('''=''' * 100)
     logger.info(__A)
     logger.info('''=''' * 100)


if __name__ == "__main__":
  main()



