


import argparse
import logging
import os
import sys
from collections import namedtuple

import torch
from modeling_bertabs import BertAbs, build_predictor
from torch.utils.data import DataLoader, SequentialSampler
from tqdm import tqdm

from transformers import BertTokenizer

from .utils_summarization import (
    CNNDMDataset,
    build_mask,
    compute_token_type_ids,
    encode_for_summarization,
    truncate_or_pad,
)


__snake_case     =		logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)


__snake_case     =		namedtuple('''Batch''', ['''document_names''', '''batch_size''', '''src''', '''segs''', '''mask_src''', '''tgt_str'''])






def 						_A						(						_lowercase				)			->				List[Any]:


     """simple docstring"""


     __UpperCamelCase				=	BertTokenizer.from_pretrained('bert-base-uncased'			,       do_lower_case=_lowercase				)
     __UpperCamelCase				=	BertAbs.from_pretrained('remi/bertabs-finetuned-extractive-abstractive-summarization'				)
     model.to(args.device				)
     model.eval()

     __UpperCamelCase				=	{
         'BOS': tokenizer.vocab['[unused0]'],
         'EOS': tokenizer.vocab['[unused1]'],
         'PAD': tokenizer.vocab['[PAD]'],
     }

     if args.compute_rouge:
          __UpperCamelCase				=	[]
          __UpperCamelCase				=	[]

          import nltk
          import rouge

          nltk.download('punkt'				)
          __UpperCamelCase				=	rouge.Rouge(
              metrics=['rouge-n', 'rouge-l']			,       max_n=2			,       limit_length=_lowercase			,       length_limit=args.beam_size			,       length_limit_type='words'			,       apply_avg=_lowercase			,       apply_best=_lowercase			,       alpha=0.5			,       weight_factor=1.2			,       stemming=_lowercase			,       )

     # these (unused) arguments are defined to keep the compatibility
     # with the legacy code and will be deleted in a next iteration.
     __UpperCamelCase				=	''
     __UpperCamelCase				=	''

     __UpperCamelCase				=	build_data_iterator(_lowercase			,       _lowercase				)
     __UpperCamelCase				=	build_predictor(_lowercase			,       _lowercase			,       _lowercase			,       _lowercase				)

     logger.info('***** Running evaluation *****'				)
     logger.info('  Number examples = %d'			,       len(data_iterator.dataset				)				)
     logger.info('  Batch size = %d'			,       args.batch_size				)
     logger.info(''				)
     logger.info('***** Beam Search parameters *****'				)
     logger.info('  Beam size = %d'			,       args.beam_size				)
     logger.info('  Minimum length = %d'			,       args.min_length				)
     logger.info('  Maximum length = %d'			,       args.max_length				)
     logger.info('  Alpha (length penalty) = %.2f'			,       args.alpha				)
     logger.info('  Trigrams %s be blocked'			,       ('will' if args.block_trigram else 'will NOT')				)

     for batch in tqdm(_lowercase				):
          __UpperCamelCase				=	predictor.translate_batch(_lowercase				)
          __UpperCamelCase				=	predictor.from_batch(_lowercase				)
          __UpperCamelCase				=	[format_summary(_lowercase				) for t in translations]
          save_summaries(_lowercase			,       args.summaries_output_dir			,       batch.document_names				)

          if args.compute_rouge:
               reference_summaries += batch.tgt_str
               generated_summaries += summaries

     if args.compute_rouge:
          __UpperCamelCase				=	rouge_evaluator.get_scores(_lowercase			,       _lowercase				)
          __UpperCamelCase				=	format_rouge_scores(_lowercase				)
          save_rouge_scores(_lowercase				)
          print(_lowercase				)






def 						_A						(						_lowercase			,       _lowercase			,       _lowercase				)			->				str:


     """simple docstring"""


     for summary, document_name in zip(_lowercase			,       _lowercase				):
          # Prepare the summary file's name
          if "." in document_name:
               __UpperCamelCase				=	'.'.join(document_name.split('.'				)[:-1]				)
               __UpperCamelCase				=	document_name.split('.'				)[-1]
               __UpperCamelCase				=	bare_document_name + '_summary.' + extension
          else:
               __UpperCamelCase				=	document_name + '_summary'

          __UpperCamelCase				=	os.path.join(_lowercase			,       _lowercase				)
          with open(_lowercase			,       'w'				) as output:
               output.write(_lowercase				)






def 						_A						(						_lowercase				)			->				Any:


     """simple docstring"""


     __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase				=	translation
     __UpperCamelCase				=	(
         raw_summary.replace('[unused0]'			,       ''				)
         .replace('[unused3]'			,       ''				)
         .replace('[PAD]'			,       ''				)
         .replace('[unused1]'			,       ''				)
         .replace(r' +'			,       ' '				)
         .replace(' [unused2] '			,       '. '				)
         .replace('[unused2]'			,       ''				)
         .strip()
     )

     return summary






def 						_A						(						_lowercase				)			->				int:


     """simple docstring"""


     return """\n
****** ROUGE SCORES ******

** ROUGE 1
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE 2
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE L
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}""".format(
         scores['rouge-1']['f']			,       scores['rouge-1']['p']			,       scores['rouge-1']['r']			,       scores['rouge-2']['f']			,       scores['rouge-2']['p']			,       scores['rouge-2']['r']			,       scores['rouge-l']['f']			,       scores['rouge-l']['p']			,       scores['rouge-l']['r']			,       )






def 						_A						(						_lowercase				)			->				Dict:


     """simple docstring"""


     with open('rouge_scores.txt'			,       'w'				) as output:
          output.write(_lowercase				)






def 						_A						(						_lowercase			,       _lowercase				)			->				List[Any]:


     """simple docstring"""


     __UpperCamelCase				=	load_and_cache_examples(_lowercase			,       _lowercase				)
     __UpperCamelCase				=	SequentialSampler(_lowercase				)

     def collate_fn(_lowercase				):
          return collate(_lowercase			,       _lowercase			,       block_size=5_12			,       device=args.device				)

     __UpperCamelCase				=	DataLoader(
         _lowercase			,       sampler=_lowercase			,       batch_size=args.batch_size			,       collate_fn=_lowercase			,       )

     return iterator






def 						_A						(						_lowercase			,       _lowercase				)			->				List[str]:


     """simple docstring"""


     __UpperCamelCase				=	CNNDMDataset(args.documents_dir				)
     return dataset






def 						_A						(						_lowercase			,       _lowercase			,       _lowercase			,       _lowercase				)			->				str:


     """simple docstring"""


     __UpperCamelCase				=	[x for x in data if not len(x[1]				) == 0]  # remove empty_files
     __UpperCamelCase				=	[name for name, _, _ in data]
     __UpperCamelCase				=	[' '.join(_lowercase				) for _, _, summary_list in data]

     __UpperCamelCase				=	[encode_for_summarization(_lowercase			,       _lowercase			,       _lowercase				) for _, story, summary in data]
     __UpperCamelCase				=	torch.tensor(
         [truncate_or_pad(_lowercase			,       _lowercase			,       tokenizer.pad_token_id				) for story, _ in encoded_text]				)
     __UpperCamelCase				=	compute_token_type_ids(_lowercase			,       tokenizer.cls_token_id				)
     __UpperCamelCase				=	build_mask(_lowercase			,       tokenizer.pad_token_id				)

     __UpperCamelCase				=	Batch(
         document_names=_lowercase			,       batch_size=len(_lowercase				)			,       src=encoded_stories.to(_lowercase				)			,       segs=encoder_token_type_ids.to(_lowercase				)			,       mask_src=encoder_mask.to(_lowercase				)			,       tgt_str=_lowercase			,       )

     return batch






def 						_A						(						_lowercase			,       _lowercase				)			->				int:


     """simple docstring"""


     __UpperCamelCase				=	summary_tokens.to('cpu'				).numpy()
     __UpperCamelCase				=	tokenizer.decode(_lowercase				)
     __UpperCamelCase				=	summary.split('.'				)
     __UpperCamelCase				=	[s + '.' for s in sentences]
     return sentences






def 						_A						(						)			->				Optional[Any]:


     """simple docstring"""


     __UpperCamelCase				=	argparse.ArgumentParser()
     parser.add_argument(
         '--documents_dir'			,       default=_lowercase			,       type=_lowercase			,       required=_lowercase			,       help='The folder where the documents to summarize are located.'			,       )
     parser.add_argument(
         '--summaries_output_dir'			,       default=_lowercase			,       type=_lowercase			,       required=_lowercase			,       help='The folder in wich the summaries should be written. Defaults to the folder where the documents are'			,       )
     parser.add_argument(
         '--compute_rouge'			,       default=_lowercase			,       type=_lowercase			,       required=_lowercase			,       help='Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.'			,       )
     # EVALUATION options
     parser.add_argument(
         '--no_cuda'			,       default=_lowercase			,       type=_lowercase			,       help='Whether to force the execution on CPU.'			,       )
     parser.add_argument(
         '--batch_size'			,       default=4			,       type=_lowercase			,       help='Batch size per GPU/CPU for training.'			,       )
     # BEAM SEARCH arguments
     parser.add_argument(
         '--min_length'			,       default=50			,       type=_lowercase			,       help='Minimum number of tokens for the summaries.'			,       )
     parser.add_argument(
         '--max_length'			,       default=2_00			,       type=_lowercase			,       help='Maixmum number of tokens for the summaries.'			,       )
     parser.add_argument(
         '--beam_size'			,       default=5			,       type=_lowercase			,       help='The number of beams to start with for each example.'			,       )
     parser.add_argument(
         '--alpha'			,       default=0.95			,       type=_lowercase			,       help='The value of alpha for the length penalty in the beam search.'			,       )
     parser.add_argument(
         '--block_trigram'			,       default=_lowercase			,       type=_lowercase			,       help='Whether to block the existence of repeating trigrams in the text generated by beam search.'			,       )
     __UpperCamelCase				=	parser.parse_args()

     # Select device (distibuted not available)
     __UpperCamelCase				=	torch.device('cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu'				)

     # Check the existence of directories
     if not args.summaries_output_dir:
          __UpperCamelCase				=	args.documents_dir

     if not documents_dir_is_valid(args.documents_dir				):
          raise FileNotFoundError(
              'We could not find the directory you specified for the documents to summarize, or it was empty. Please'
              ' specify a valid path.'				)
     os.makedirs(args.summaries_output_dir			,       exist_ok=_lowercase				)

     evaluate(_lowercase				)






def 						_A						(						_lowercase				)			->				Optional[int]:


     """simple docstring"""


     if not os.path.exists(_lowercase				):
          return False

     __UpperCamelCase				=	os.listdir(_lowercase				)
     if len(_lowercase				) == 0:
          return False

     return True


if __name__ == "__main__":
    main()






import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand



class 					__lowerCamelCase	(_a		):







       @staticmethod
       def 						snake_case_					(	A_:				ArgumentParser			):






            '''simple docstring'''




            __UpperCamelCase				=	parser.add_parser('login',help='Log in using the same credentials as on huggingface.co'			)
            login_parser.set_defaults(func=lambda A_			: LoginCommand(A_			)			)
            __UpperCamelCase				=	parser.add_parser('whoami',help='Find out which huggingface.co account you are logged in as.'			)
            whoami_parser.set_defaults(func=lambda A_			: WhoamiCommand(A_			)			)
            __UpperCamelCase				=	parser.add_parser('logout',help='Log out'			)
            logout_parser.set_defaults(func=lambda A_			: LogoutCommand(A_			)			)

            # new system: git-based repo system
            __UpperCamelCase				=	parser.add_parser(
                'repo',help='Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.',)
            __UpperCamelCase				=	repo_parser.add_subparsers(
                help='Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands'			)
            __UpperCamelCase				=	repo_subparsers.add_parser(
                'create',help='Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co'			)
            repo_create_parser.add_argument(
                'name',type=A_,help='Name for your model\'s repo. Will be namespaced under your username to build the model id.',)
            repo_create_parser.add_argument('--organization',type=A_,help='Optional: organization namespace.'			)
            repo_create_parser.add_argument('-y','--yes',action='store_true',help='Optional: answer Yes to the prompt'			)
            repo_create_parser.set_defaults(func=lambda A_			: RepoCreateCommand(A_			)			)



class 					__lowerCamelCase	:
       _lowercase    							=      """\u001b[1m"""
       _lowercase    							=      """\u001b[31m"""
       _lowercase    							=      """\u001b[90m"""
       _lowercase    							=      """\u001b[0m"""







       @classmethod
       def 						snake_case_					(	cls:				Optional[Any],A_:				Tuple			):






            '''simple docstring'''




            return F'''{cls._bold}{s}{cls._reset}'''







       @classmethod
       def 						snake_case_					(	cls:				Any,A_:				Union[str, Any]			):






            '''simple docstring'''




            return F'''{cls._bold}{cls._red}{s}{cls._reset}'''







       @classmethod
       def 						snake_case_					(	cls:				Optional[Any],A_:				Dict			):






            '''simple docstring'''




            return F'''{cls._gray}{s}{cls._reset}'''






def 						_A						(						_lowercase			,       _lowercase				)			->				str:


     """simple docstring"""


     __UpperCamelCase				=	[max(len(str(_lowercase				)				) for x in col				) for col in zip(*_lowercase			,       _lowercase				)]
     __UpperCamelCase				=	('{{:{}}} ' * len(_lowercase				)).format(*_lowercase				)
     __UpperCamelCase				=	[]
     lines.append(row_format.format(*_lowercase				)				)
     lines.append(row_format.format(*['-' * w for w in col_widths]				)				)
     for row in rows:
          lines.append(row_format.format(*_lowercase				)				)
     return "\n".join(_lowercase				)



class 					__lowerCamelCase	:







       def __init__(	self:				List[Any],A_:				Any			):






            '''simple docstring'''




            __UpperCamelCase				=	args



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'ERROR! `huggingface-cli login` uses an outdated login mechanism '
                    'that is not compatible with the Hugging Face Hub backend anymore. '
                    'Please use `huggingface-cli login instead.'			)			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '
                    '`huggingface-cli whoami` instead.'			)			)
            __UpperCamelCase				=	HfFolder.get_token()
            if token is None:
                 print('Not logged in'			)
                 exit()
            try:
                 __UpperCamelCase,  __UpperCamelCase				=	whoami(A_			)
                 print(A_			)
                 if orgs:
                      print(ANSI.bold('orgs: '			),','.join(A_			)			)
            except HTTPError as e:
                 print(A_			)
                 print(ANSI.red(e.response.text			)			)
                 exit(1			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'ERROR! `transformers-cli logout` uses an outdated logout mechanism '
                    'that is not compatible with the Hugging Face Hub backend anymore. '
                    'Please use `huggingface-cli logout instead.'			)			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'WARNING! Managing repositories through transformers-cli is deprecated. '
                    'Please use `huggingface-cli` instead.'			)			)
            __UpperCamelCase				=	HfFolder.get_token()
            if token is None:
                 print('Not logged in'			)
                 exit(1			)
            try:
                 __UpperCamelCase				=	subprocess.check_output(['git', '--version']			).decode('utf-8'			)
                 print(ANSI.gray(stdout.strip()			)			)
            except FileNotFoundError:
                 print('Looks like you do not have git installed, please install.'			)

            try:
                 __UpperCamelCase				=	subprocess.check_output(['git-lfs', '--version']			).decode('utf-8'			)
                 print(ANSI.gray(stdout.strip()			)			)
            except FileNotFoundError:
                 print(
                     ANSI.red(
                         'Looks like you do not have git-lfs installed, please install.'
                         ' You can install from https://git-lfs.github.com/.'
                         ' Then run `git lfs install` (you only have to do this once).'			)			)
            print(''			)

            __UpperCamelCase,  __UpperCamelCase				=	whoami(A_			)
            __UpperCamelCase				=	self.args.organization if self.args.organization is not None else user
            __UpperCamelCase				=	F'''{namespace}/{self.args.name}'''
            print(F'''You are about to create {ANSI.bold(A_			)}'''			)

            if not self.args.yes:
                 __UpperCamelCase				=	input('Proceed? [Y/n] '			).lower()
                 if not (choice == "" or choice == "y" or choice == "yes"):
                      print('Abort'			)
                      exit()
            try:
                 __UpperCamelCase				=	create_repo(A_,name=self.args.name,organization=self.args.organization			)
            except HTTPError as e:
                 print(A_			)
                 print(ANSI.red(e.response.text			)			)
                 exit(1			)
            print('\nYour repo now lives at:'			)
            print(F'''  {ANSI.bold(A_			)}'''			)
            print('\nYou can clone it locally with the command below, and commit/push as usual.'			)
            print(F'''\n  git clone {url}'''			)
            print(''			)


