





import argparse
import logging
import os
import sys
from collections import namedtuple

import torch
from modeling_bertabs import BertAbs, build_predictor
from torch.utils.data import DataLoader, SequentialSampler
from tqdm import tqdm

from transformers import BertTokenizer

from .utils_summarization import (
    CNNDMDataset,
    build_mask,
    compute_token_type_ids,
    encode_for_summarization,
    truncate_or_pad,
)


_lowerCAmelCase  :				Optional[int]						=				logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)


_lowerCAmelCase  :				List[Any]						=				namedtuple("Batch", ["document_names", "batch_size", "src", "segs", "mask_src", "tgt_str"])



def   lowerCAmelCase						(						_lowerCAmelCase							:				Tuple		):




 """simple docstring"""







 UpperCAmelCase__								=  BertTokenizer.from_pretrained("bert-base-uncased"  ,							do_lower_case=lowercase_		)
 UpperCAmelCase__								=  BertAbs.from_pretrained("remi/bertabs-finetuned-extractive-abstractive-summarization"		)
 model.to(args.device		)
 model.eval()

 UpperCAmelCase__								=  {
     "BOS": tokenizer.vocab["[unused0]"],
     "EOS": tokenizer.vocab["[unused1]"],
     "PAD": tokenizer.vocab["[PAD]"],
 }

 if args.compute_rouge:
  UpperCAmelCase__								=  []
  UpperCAmelCase__								=  []

  import nltk
  import rouge

  nltk.download("punkt"		)
  UpperCAmelCase__								=  rouge.Rouge(
      metrics=["rouge-n", "rouge-l"]  ,							max_n=2  ,							limit_length=lowercase_  ,							length_limit=args.beam_size  ,							length_limit_type="words"  ,							apply_avg=lowercase_  ,							apply_best=lowercase_  ,							alpha=0.5  ,							weight_factor=1.2  ,							stemming=lowercase_  ,							)

 # these (unused) arguments are defined to keep the compatibility
 # with the legacy code and will be deleted in a next iteration.
 UpperCAmelCase__								=  ""
 UpperCAmelCase__								=  ""

 UpperCAmelCase__								=  build_data_iterator(lowercase_  ,							lowercase_		)
 UpperCAmelCase__								=  build_predictor(lowercase_  ,							lowercase_  ,							lowercase_  ,							lowercase_		)

 logger.info("***** Running evaluation *****"		)
 logger.info("  Number examples = %d"  ,							len(data_iterator.dataset		)		)
 logger.info("  Batch size = %d"  ,							args.batch_size		)
 logger.info(""		)
 logger.info("***** Beam Search parameters *****"		)
 logger.info("  Beam size = %d"  ,							args.beam_size		)
 logger.info("  Minimum length = %d"  ,							args.min_length		)
 logger.info("  Maximum length = %d"  ,							args.max_length		)
 logger.info("  Alpha (length penalty) = %.2f"  ,							args.alpha		)
 logger.info("  Trigrams %s be blocked"  ,							("will" if args.block_trigram else "will NOT")		)

 for batch in tqdm(lowercase_		):
  UpperCAmelCase__								=  predictor.translate_batch(lowercase_		)
  UpperCAmelCase__								=  predictor.from_batch(lowercase_		)
  UpperCAmelCase__								=  [format_summary(lowercase_		) for t in translations]
  save_summaries(lowercase_  ,							args.summaries_output_dir  ,							batch.document_names		)

  if args.compute_rouge:
   reference_summaries += batch.tgt_str
   generated_summaries += summaries

 if args.compute_rouge:
  UpperCAmelCase__								=  rouge_evaluator.get_scores(lowercase_  ,							lowercase_		)
  UpperCAmelCase__								=  format_rouge_scores(lowercase_		)
  save_rouge_scores(lowercase_		)
  print(lowercase_		)



def   lowerCAmelCase						(						_lowerCAmelCase							:				Dict  ,							_lowerCAmelCase							:				str  ,							_lowerCAmelCase							:				Tuple		):




 """simple docstring"""







 for summary, document_name in zip(lowercase_  ,							lowercase_		):
  # Prepare the summary file's name
  if "." in document_name:
   UpperCAmelCase__								=  ".".join(document_name.split("."		)[:-1]		)
   UpperCAmelCase__								=  document_name.split("."		)[-1]
   UpperCAmelCase__								=  bare_document_name + "_summary." + extension
  else:
   UpperCAmelCase__								=  document_name + "_summary"

  UpperCAmelCase__								=  os.path.join(lowercase_  ,							lowercase_		)
  with open(lowercase_  ,							"w"		) as output:
   output.write(lowercase_		)



def   lowerCAmelCase						(						_lowerCAmelCase							:				Union[str, Any]		):




 """simple docstring"""







 UpperCAmelCase__	,	UpperCAmelCase__	,	UpperCAmelCase__								=  translation
 UpperCAmelCase__								=  (
     raw_summary.replace("[unused0]"  ,							""		)
     .replace("[unused3]"  ,							""		)
     .replace("[PAD]"  ,							""		)
     .replace("[unused1]"  ,							""		)
     .replace(R" +"  ,							" "		)
     .replace(" [unused2] "  ,							". "		)
     .replace("[unused2]"  ,							""		)
     .strip()
 )

 return summary



def   lowerCAmelCase						(						_lowerCAmelCase							:				Optional[Any]		):




 """simple docstring"""







 return """\n
****** ROUGE SCORES ******

** ROUGE 1
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE 2
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE L
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}""".format(
     scores["rouge-1"]["f"]  ,							scores["rouge-1"]["p"]  ,							scores["rouge-1"]["r"]  ,							scores["rouge-2"]["f"]  ,							scores["rouge-2"]["p"]  ,							scores["rouge-2"]["r"]  ,							scores["rouge-l"]["f"]  ,							scores["rouge-l"]["p"]  ,							scores["rouge-l"]["r"]  ,							)



def   lowerCAmelCase						(						_lowerCAmelCase							:				Any		):




 """simple docstring"""







 with open("rouge_scores.txt"  ,							"w"		) as output:
  output.write(lowercase_		)



def   lowerCAmelCase						(						_lowerCAmelCase							:				Dict  ,							_lowerCAmelCase							:				int		):




 """simple docstring"""







 UpperCAmelCase__								=  load_and_cache_examples(lowercase_  ,							lowercase_		)
 UpperCAmelCase__								=  SequentialSampler(lowercase_		)

 def collate_fn(_lowerCAmelCase							:				Any		):
  return collate(lowercase_  ,							lowercase_  ,							block_size=512  ,							device=args.device		)

 UpperCAmelCase__								=  DataLoader(
     lowercase_  ,							sampler=lowercase_  ,							batch_size=args.batch_size  ,							collate_fn=lowercase_  ,							)

 return iterator



def   lowerCAmelCase						(						_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				Union[str, Any]		):




 """simple docstring"""







 UpperCAmelCase__								=  CNNDMDataset(args.documents_dir		)
 return dataset



def   lowerCAmelCase						(						_lowerCAmelCase							:				Tuple  ,							_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				List[str]  ,							_lowerCAmelCase							:				List[Any]		):




 """simple docstring"""







 UpperCAmelCase__								=  [x for x in data if not len(x[1]		) == 0]  # remove empty_files
 UpperCAmelCase__								=  [name for name, _, _ in data]
 UpperCAmelCase__								=  [" ".join(lowercase_		) for _, _, summary_list in data]

 UpperCAmelCase__								=  [encode_for_summarization(lowercase_  ,							lowercase_  ,							lowercase_		) for _, story, summary in data]
 UpperCAmelCase__								=  torch.tensor(
     [truncate_or_pad(lowercase_  ,							lowercase_  ,							tokenizer.pad_token_id		) for story, _ in encoded_text]		)
 UpperCAmelCase__								=  compute_token_type_ids(lowercase_  ,							tokenizer.cls_token_id		)
 UpperCAmelCase__								=  build_mask(lowercase_  ,							tokenizer.pad_token_id		)

 UpperCAmelCase__								=  Batch(
     document_names=lowercase_  ,							batch_size=len(lowercase_		)  ,							src=encoded_stories.to(lowercase_		)  ,							segs=encoder_token_type_ids.to(lowercase_		)  ,							mask_src=encoder_mask.to(lowercase_		)  ,							tgt_str=lowercase_  ,							)

 return batch



def   lowerCAmelCase						(						_lowerCAmelCase							:				List[str]  ,							_lowerCAmelCase							:				Union[str, Any]		):




 """simple docstring"""







 UpperCAmelCase__								=  summary_tokens.to("cpu"		).numpy()
 UpperCAmelCase__								=  tokenizer.decode(lowercase_		)
 UpperCAmelCase__								=  summary.split("."		)
 UpperCAmelCase__								=  [s + "." for s in sentences]
 return sentences



def   lowerCAmelCase						(						):




 """simple docstring"""







 UpperCAmelCase__								=  argparse.ArgumentParser()
 parser.add_argument(
     "--documents_dir"  ,							default=lowercase_  ,							type=lowercase_  ,							required=lowercase_  ,							help="The folder where the documents to summarize are located."  ,							)
 parser.add_argument(
     "--summaries_output_dir"  ,							default=lowercase_  ,							type=lowercase_  ,							required=lowercase_  ,							help="The folder in wich the summaries should be written. Defaults to the folder where the documents are"  ,							)
 parser.add_argument(
     "--compute_rouge"  ,							default=lowercase_  ,							type=lowercase_  ,							required=lowercase_  ,							help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset."  ,							)
 # EVALUATION options
 parser.add_argument(
     "--no_cuda"  ,							default=lowercase_  ,							type=lowercase_  ,							help="Whether to force the execution on CPU."  ,							)
 parser.add_argument(
     "--batch_size"  ,							default=4  ,							type=lowercase_  ,							help="Batch size per GPU/CPU for training."  ,							)
 # BEAM SEARCH arguments
 parser.add_argument(
     "--min_length"  ,							default=50  ,							type=lowercase_  ,							help="Minimum number of tokens for the summaries."  ,							)
 parser.add_argument(
     "--max_length"  ,							default=200  ,							type=lowercase_  ,							help="Maixmum number of tokens for the summaries."  ,							)
 parser.add_argument(
     "--beam_size"  ,							default=5  ,							type=lowercase_  ,							help="The number of beams to start with for each example."  ,							)
 parser.add_argument(
     "--alpha"  ,							default=0.95  ,							type=lowercase_  ,							help="The value of alpha for the length penalty in the beam search."  ,							)
 parser.add_argument(
     "--block_trigram"  ,							default=lowercase_  ,							type=lowercase_  ,							help="Whether to block the existence of repeating trigrams in the text generated by beam search."  ,							)
 UpperCAmelCase__								=  parser.parse_args()

 # Select device (distibuted not available)
 UpperCAmelCase__								=  torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu"		)

 # Check the existence of directories
 if not args.summaries_output_dir:
  UpperCAmelCase__								=  args.documents_dir

 if not documents_dir_is_valid(args.documents_dir		):
  raise FileNotFoundError(
      "We could not find the directory you specified for the documents to summarize, or it was empty. Please"
      " specify a valid path."		)
 os.makedirs(args.summaries_output_dir  ,							exist_ok=lowercase_		)

 evaluate(lowercase_		)



def   lowerCAmelCase						(						_lowerCAmelCase							:				Tuple		):




 """simple docstring"""







 if not os.path.exists(lowercase_		):
  return False

 UpperCAmelCase__								=  os.listdir(lowercase_		)
 if len(lowercase_		) == 0:
  return False

 return True


if __name__ == "__main__":
  main()











import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      YosoForMaskedLM,
      YosoForMultipleChoice,
      YosoForQuestionAnswering,
      YosoForSequenceClassification,
      YosoForTokenClassification,
      YosoModel,
  )
  from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
class      _UpperCamelCase :
      def __init__(     self				:int	, lowerCamelCase				:int	, lowerCamelCase				:Optional[Any]=13	, lowerCamelCase				:Optional[Any]=7	, lowerCamelCase				:Union[str, Any]=True	, lowerCamelCase				:Tuple=True	, lowerCamelCase				:Optional[Any]=True	, lowerCamelCase				:str=True	, lowerCamelCase				:Any=99	, lowerCamelCase				:Optional[int]=32	, lowerCamelCase				:Any=5	, lowerCamelCase				:List[Any]=4	, lowerCamelCase				:Optional[Any]=37	, lowerCamelCase				:Tuple="gelu"	, lowerCamelCase				:Optional[int]=0.1	, lowerCamelCase				:Any=0.1	, lowerCamelCase				:List[Any]=512	, lowerCamelCase				:Tuple=16	, lowerCamelCase				:Optional[int]=2	, lowerCamelCase				:str=0.02	, lowerCamelCase				:Dict=3	, lowerCamelCase				:Union[str, Any]=4	, lowerCamelCase				:Union[str, Any]=None	, )       ->     List[Any]:
       UpperCAmelCase__								=  parent
       UpperCAmelCase__								=  batch_size
       UpperCAmelCase__								=  seq_length
       UpperCAmelCase__								=  is_training
       UpperCAmelCase__								=  use_input_mask
       UpperCAmelCase__								=  use_token_type_ids
       UpperCAmelCase__								=  use_labels
       UpperCAmelCase__								=  vocab_size
       UpperCAmelCase__								=  hidden_size
       UpperCAmelCase__								=  num_hidden_layers
       UpperCAmelCase__								=  num_attention_heads
       UpperCAmelCase__								=  intermediate_size
       UpperCAmelCase__								=  hidden_act
       UpperCAmelCase__								=  hidden_dropout_prob
       UpperCAmelCase__								=  attention_probs_dropout_prob
       UpperCAmelCase__								=  max_position_embeddings
       UpperCAmelCase__								=  type_vocab_size
       UpperCAmelCase__								=  type_sequence_label_size
       UpperCAmelCase__								=  initializer_range
       UpperCAmelCase__								=  num_labels
       UpperCAmelCase__								=  num_choices
       UpperCAmelCase__								=  scope
      def 		UpperCAmelCase_  (     self				:Union[str, Any] )       ->     Any:
       UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	, self.vocab_size )

       UpperCAmelCase__								=  None
       if self.use_input_mask:
        UpperCAmelCase__								=  random_attention_mask([self.batch_size, self.seq_length] )

       UpperCAmelCase__								=  None
       if self.use_token_type_ids:
        UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	, self.type_vocab_size )

       UpperCAmelCase__								=  None
       UpperCAmelCase__								=  None
       UpperCAmelCase__								=  None
       if self.use_labels:
        UpperCAmelCase__								=  ids_tensor([self.batch_size]	, self.type_sequence_label_size )
        UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	, self.num_labels )
        UpperCAmelCase__								=  ids_tensor([self.batch_size]	, self.num_choices )

       UpperCAmelCase__								=  self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      def 		UpperCAmelCase_  (     self				:List[Any] )       ->     Optional[Any]:
       return YosoConfig(
           vocab_size=self.vocab_size	, hidden_size=self.hidden_size	, num_hidden_layers=self.num_hidden_layers	, num_attention_heads=self.num_attention_heads	, intermediate_size=self.intermediate_size	, hidden_act=self.hidden_act	, hidden_dropout_prob=self.hidden_dropout_prob	, attention_probs_dropout_prob=self.attention_probs_dropout_prob	, max_position_embeddings=self.max_position_embeddings	, type_vocab_size=self.type_vocab_size	, is_decoder=lowerCamelCase	, initializer_range=self.initializer_range	, )
      def 		UpperCAmelCase_  (     self				:int )       ->     Any:
       UpperCAmelCase__								=  self.get_config()
       UpperCAmelCase__								=  300
       return config
      def 		UpperCAmelCase_  (     self				:List[str] )       ->     Tuple:
       (
           (
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	
       )								=  self.prepare_config_and_inputs()

       UpperCAmelCase__								=  True
       UpperCAmelCase__								=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size] )
       UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	, vocab_size=2 )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )
      def 		UpperCAmelCase_  (     self				:Any	, lowerCamelCase				:List[str]	, lowerCamelCase				:List[str]	, lowerCamelCase				:str	, lowerCamelCase				:Union[str, Any]	, lowerCamelCase				:Optional[int]	, lowerCamelCase				:Any	, lowerCamelCase				:Tuple )       ->     List[Any]:
       UpperCAmelCase__								=  YosoModel(config=lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase )
       UpperCAmelCase__								=  model(lowerCamelCase	, token_type_ids=lowerCamelCase )
       UpperCAmelCase__								=  model(lowerCamelCase )
       self.parent.assertEqual(result.last_hidden_state.shape	, (self.batch_size, self.seq_length, self.hidden_size) )
      def 		UpperCAmelCase_  (     self				:Any	, lowerCamelCase				:str	, lowerCamelCase				:Dict	, lowerCamelCase				:Any	, lowerCamelCase				:Any	, lowerCamelCase				:int	, lowerCamelCase				:Dict	, lowerCamelCase				:Any	, lowerCamelCase				:Union[str, Any]	, lowerCamelCase				:List[Any]	, )       ->     Optional[int]:
       UpperCAmelCase__								=  True
       UpperCAmelCase__								=  YosoModel(lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(
           lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, encoder_hidden_states=lowerCamelCase	, encoder_attention_mask=lowerCamelCase	, )
       UpperCAmelCase__								=  model(
           lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, encoder_hidden_states=lowerCamelCase	, )
       UpperCAmelCase__								=  model(lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase )
       self.parent.assertEqual(result.last_hidden_state.shape	, (self.batch_size, self.seq_length, self.hidden_size) )
      def 		UpperCAmelCase_  (     self				:Tuple	, lowerCamelCase				:List[Any]	, lowerCamelCase				:Tuple	, lowerCamelCase				:List[str]	, lowerCamelCase				:Optional[Any]	, lowerCamelCase				:Tuple	, lowerCamelCase				:Optional[Any]	, lowerCamelCase				:int )       ->     Optional[int]:
       UpperCAmelCase__								=  YosoForMaskedLM(config=lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, labels=lowerCamelCase )
       self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.seq_length, self.vocab_size) )
      def 		UpperCAmelCase_  (     self				:Dict	, lowerCamelCase				:str	, lowerCamelCase				:Dict	, lowerCamelCase				:List[Any]	, lowerCamelCase				:Optional[int]	, lowerCamelCase				:Union[str, Any]	, lowerCamelCase				:int	, lowerCamelCase				:Optional[Any] )       ->     Any:
       UpperCAmelCase__								=  YosoForQuestionAnswering(config=lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(
           lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, start_positions=lowerCamelCase	, end_positions=lowerCamelCase	, )
       self.parent.assertEqual(result.start_logits.shape	, (self.batch_size, self.seq_length) )
       self.parent.assertEqual(result.end_logits.shape	, (self.batch_size, self.seq_length) )
      def 		UpperCAmelCase_  (     self				:Optional[Any]	, lowerCamelCase				:Optional[Any]	, lowerCamelCase				:Any	, lowerCamelCase				:List[str]	, lowerCamelCase				:Dict	, lowerCamelCase				:str	, lowerCamelCase				:Tuple	, lowerCamelCase				:int )       ->     Dict:
       UpperCAmelCase__								=  self.num_labels
       UpperCAmelCase__								=  YosoForSequenceClassification(lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, labels=lowerCamelCase )
       self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.num_labels) )
      def 		UpperCAmelCase_  (     self				:Tuple	, lowerCamelCase				:str	, lowerCamelCase				:int	, lowerCamelCase				:Optional[Any]	, lowerCamelCase				:List[str]	, lowerCamelCase				:Dict	, lowerCamelCase				:Optional[int]	, lowerCamelCase				:str )       ->     Dict:
       UpperCAmelCase__								=  self.num_labels
       UpperCAmelCase__								=  YosoForTokenClassification(config=lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  model(lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, labels=lowerCamelCase )
       self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.seq_length, self.num_labels) )
      def 		UpperCAmelCase_  (     self				:Optional[int]	, lowerCamelCase				:Tuple	, lowerCamelCase				:Tuple	, lowerCamelCase				:int	, lowerCamelCase				:Tuple	, lowerCamelCase				:List[str]	, lowerCamelCase				:Optional[Any]	, lowerCamelCase				:List[str] )       ->     List[Any]:
       UpperCAmelCase__								=  self.num_choices
       UpperCAmelCase__								=  YosoForMultipleChoice(config=lowerCamelCase )
       model.to(lowerCamelCase )
       model.eval()
       UpperCAmelCase__								=  input_ids.unsqueeze(1 ).expand(-1	, self.num_choices	, -1 ).contiguous()
       UpperCAmelCase__								=  token_type_ids.unsqueeze(1 ).expand(-1	, self.num_choices	, -1 ).contiguous()
       UpperCAmelCase__								=  input_mask.unsqueeze(1 ).expand(-1	, self.num_choices	, -1 ).contiguous()
       UpperCAmelCase__								=  model(
           lowerCamelCase	, attention_mask=lowerCamelCase	, token_type_ids=lowerCamelCase	, labels=lowerCamelCase	, )
       self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.num_choices) )







      def 		UpperCAmelCase_  (     self				:int )       ->     List[Any]:
       UpperCAmelCase__								=  self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	(
           UpperCAmelCase__
       )	,	
       )								=  config_and_inputs
       UpperCAmelCase__								=  {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict



@require_torch
class      _UpperCamelCase (			lowerCAmelCase		,    lowerCAmelCase		,    unittest.TestCase						):
      UpperCAmelCase_            =						(
          (
              YosoModel,
              YosoForMaskedLM,
              YosoForMultipleChoice,
              YosoForQuestionAnswering,
              YosoForSequenceClassification,
              YosoForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      UpperCAmelCase_            =						False
      UpperCAmelCase_            =						False
      UpperCAmelCase_            =						False

      UpperCAmelCase_            =						()
      UpperCAmelCase_            =						(
          {
              """feature-extraction""": YosoModel,
              """fill-mask""": YosoForMaskedLM,
              """question-answering""": YosoForQuestionAnswering,
              """text-classification""": YosoForSequenceClassification,
              """token-classification""": YosoForTokenClassification,
              """zero-shot""": YosoForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      def 		UpperCAmelCase_  (     self				:Any )       ->     Optional[Any]:
       UpperCAmelCase__								=  YosoModelTester(self )
       UpperCAmelCase__								=  ConfigTester(self	, config_class=lowerCamelCase	, hidden_size=37 )
      def 		UpperCAmelCase_  (     self				:List[Any] )       ->     List[str]:
       self.config_tester.run_common_tests()
      def 		UpperCAmelCase_  (     self				:Optional[int] )       ->     List[Any]:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Tuple )       ->     Optional[Any]:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
        UpperCAmelCase__								=  type
        self.model_tester.create_and_check_model(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Tuple )       ->     Optional[int]:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Union[str, Any] )       ->     Dict:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:str )       ->     Optional[Any]:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Any )       ->     List[str]:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Any )       ->     str:
       UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*lowerCamelCase )
      @slow
      def 		UpperCAmelCase_  (     self				:List[Any] )       ->     Optional[int]:
       for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        UpperCAmelCase__								=  YosoModel.from_pretrained(lowerCamelCase )
        self.assertIsNotNone(lowerCamelCase )







      def 		UpperCAmelCase_  (     self				:int )       ->     List[str]:
       return



@require_torch
class      _UpperCamelCase (			unittest.TestCase						):
      @slow
      def 		UpperCAmelCase_  (     self				:Optional[Any] )       ->     Optional[int]:
       UpperCAmelCase__								=  YosoModel.from_pretrained("uw-madison/yoso-4096" )
       UpperCAmelCase__								=  torch.tensor([[0, 1, 2, 3, 4, 5]] )

       with torch.no_grad():
        UpperCAmelCase__								=  model(lowerCamelCase )[0]

       UpperCAmelCase__								=  torch.Size((1, 6, 768) )
       self.assertEqual(output.shape	, lowerCamelCase )

       UpperCAmelCase__								=  torch.tensor(
           [[[-0.06_11, 0.12_42, 0.08_40], [0.02_80, -0.00_48, 0.11_25], [0.01_06, 0.02_26, 0.07_51]]] )

       self.assertTrue(torch.allclose(output[:, :3, :3]	, lowerCamelCase	, atol=1e-4 ) )
      @slow
      def 		UpperCAmelCase_  (     self				:Optional[Any] )       ->     List[Any]:
       UpperCAmelCase__								=  YosoForMaskedLM.from_pretrained("uw-madison/yoso-4096" )
       UpperCAmelCase__								=  torch.tensor([[0, 1, 2, 3, 4, 5]] )

       with torch.no_grad():
        UpperCAmelCase__								=  model(lowerCamelCase )[0]

       UpperCAmelCase__								=  5_0265

       UpperCAmelCase__								=  torch.Size((1, 6, vocab_size) )
       self.assertEqual(output.shape	, lowerCamelCase )

       UpperCAmelCase__								=  torch.tensor(
           [[[-2.13_13, -3.72_85, -2.24_07], [-2.70_47, -3.33_14, -2.64_08], [0.06_29, -2.51_66, -0.33_56]]] )

       self.assertTrue(torch.allclose(output[:, :3, :3]	, lowerCamelCase	, atol=1e-4 ) )







      @slow
      def 		UpperCAmelCase_  (     self				:Any )       ->     str:
       UpperCAmelCase__								=  YosoForMaskedLM.from_pretrained("uw-madison/yoso-4096" )
       UpperCAmelCase__								=  torch.arange(4096 ).unsqueeze(0 )

       with torch.no_grad():
        UpperCAmelCase__								=  model(lowerCamelCase )[0]

       UpperCAmelCase__								=  5_0265

       UpperCAmelCase__								=  torch.Size((1, 4096, vocab_size) )
       self.assertEqual(output.shape	, lowerCamelCase )

       UpperCAmelCase__								=  torch.tensor(
           [[[-2.39_14, -4.37_42, -5.09_56], [-4.09_88, -4.23_84, -7.04_06], [-3.14_27, -3.71_92, -6.68_00]]] )

       self.assertTrue(torch.allclose(output[:, :3, :3]	, lowerCamelCase	, atol=1e-4 ) )





