


import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
 from .tokenization_llama import LlamaTokenizer
else:
 lowerCamelCase			 =	None

lowerCamelCase			 =	logging.get_logger(__name__)
lowerCamelCase			 =	{'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase ,    lowerCamelCase			 =	'[INST]', '[/INST]'
lowerCamelCase ,    lowerCamelCase			 =	'<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
lowerCamelCase			 =	'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'




class A  (       UpperCamelCase_					):
      UpperCamelCase__							:  Optional[Any]				=VOCAB_FILES_NAMES
      UpperCamelCase__							:  Optional[Any]				=LlamaTokenizer
      UpperCamelCase__							:  Any				="left"
      UpperCamelCase__							:  Union[str, Any]				=["input_ids", "attention_mask"]






      def __init__( self						:					Tuple      ,       lowercase_						:					Any=None      ,       lowercase_						:					str=None      ,       lowercase_						:					List[str]=False      ,       lowercase_						:					str="<unk>"      ,       lowercase_						:					Optional[Any]="<s>"      ,       lowercase_						:					int="</s>"      ,       lowercase_						:					Optional[Any]=True      ,       lowercase_						:					Union[str, Any]=False      ,       **lowercase_						:					Any      ,       ) ->						Tuple:
             """simple docstring"""




             super().__init__(
                 vocab_file=__A      ,       tokenizer_file=__A      ,       clean_up_tokenization_spaces=__A      ,       unk_token=__A      ,       bos_token=__A      ,       eos_token=__A      ,       **__A      ,       )
             _lowerCamelCase	:		Any          =add_bos_token
             _lowerCamelCase	:		Optional[Any]          =add_eos_token
             self.update_post_processor()

             _lowerCamelCase	:		List[str]          =vocab_file
             _lowerCamelCase	:		Dict          =False if not self.vocab_file else True






      def 	lowerCamelCase	( self						:					Any			) ->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Union[str, Any]          =self.bos_token
             _lowerCamelCase	:		List[str]          =self.bos_token_id

             _lowerCamelCase	:		str          =self.eos_token
             _lowerCamelCase	:		List[Any]          =self.eos_token_id

             _lowerCamelCase	:		Dict          =F'''{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}'''
             _lowerCamelCase	:		Any          =F'''{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}'''

             _lowerCamelCase	:		Tuple          =[]
             if self.add_bos_token:
                    special_tokens.append((bos, bos_token_id)			)
             if self.add_eos_token:
                    special_tokens.append((eos, eos_token_id)			)
             _lowerCamelCase	:		Any          =processors.TemplateProcessing(
                 single=__A      ,       pair=__A      ,       special_tokens=__A			)






      @property
      def 	lowerCamelCase	( self						:					List[str]			) ->						Optional[int]:
             """simple docstring"""




             return self._add_eos_token






      @property
      def 	lowerCamelCase	( self						:					Optional[Any]			) ->						Dict:
             """simple docstring"""




             return self._add_bos_token






      @add_eos_token.setter
      def 	lowerCamelCase	( self						:					Optional[int]      ,       lowercase_						:					List[str]			) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		Any          =value
             self.update_post_processor()






      @add_bos_token.setter
      def 	lowerCamelCase	( self						:					Union[str, Any]      ,       lowercase_						:					Optional[int]			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		int          =value
             self.update_post_processor()






      def 	lowerCamelCase	( self						:					Union[str, Any]      ,       lowercase_						:					Dict      ,       lowercase_						:					Tuple = None			) ->						Any:
             """simple docstring"""




             if not self.can_save_slow_tokenizer:
                    raise ValueError(
                        'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                        'tokenizer.'			)

             if not os.path.isdir(__A			):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
                    return
             _lowerCamelCase	:		int          =os.path.join(
                 __A      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

             if os.path.abspath(self.vocab_file			) != os.path.abspath(__A			):
                    copyfile(self.vocab_file      ,       __A			)

             return (out_vocab_file,)






      def 	lowerCamelCase	( self						:					List[Any]      ,       lowercase_						:					Optional[Any]			) ->						Tuple:
             """simple docstring"""




             _lowerCamelCase	:		Dict          =list(conversation.iter_texts()			)
             if not all([is_user for is_user, msg in dialogue[::2]]			) or not all(
                 [not is_user for is_user, msg in dialogue[1::2]]			):
                    raise ValueError(
                        'The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'			)

             _lowerCamelCase	:		Union[str, Any]          =[]
             if len(conversation.past_user_inputs			) > 0:
                    if not conversation.past_user_inputs[0].startswith(__A			) or E_SYS not in conversation.past_user_inputs[0]:
                           _lowerCamelCase	:		List[str]          =(
                               B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                           )
             elif not dialogue[0][1].startswith(__A			) or E_SYS not in dialogue[0][1]:
                    _lowerCamelCase	:		List[Any]          =(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

             dialog_tokens += sum(
                 [
                     [self.bos_token_id]
                     + self.encode(
                         F'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''      ,       add_special_tokens=__A			)
                     + [self.eos_token_id]
                     for prompt, answer in zip(dialogue[::2]      ,       dialogue[1::2]			)
                 ]      ,       []      ,       )
             if not (dialogue[-1][0]):
                    raise ValueError(F'''Last message must be from user, got {dialogue[-1]['role']}'''			)
             dialog_tokens += [self.bos_token_id] + self.encode(
                 F'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''      ,       add_special_tokens=__A			)
             return dialog_tokens






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowerCamelCase						 =	logging.get_logger(__name__)

lowerCamelCase						 =	{
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}




class A  (       UpperCamelCase_					):
      UpperCamelCase__							:  int				='bigbird_pegasus'
      UpperCamelCase__							:  Tuple				=['past_key_values']
      UpperCamelCase__							:  str				={
          'num_attention_heads': 'encoder_attention_heads',
          'hidden_size': 'd_model',
          'attention_probs_dropout_prob': 'attention_dropout',
      }






      def __init__( self						:					Tuple      ,       lowercase_						:					int=9_6103      ,       lowercase_						:					Optional[Any]=4096      ,       lowercase_						:					Tuple=16      ,       lowercase_						:					int=4096      ,       lowercase_						:					Any=16      ,       lowercase_						:					str=16      ,       lowercase_						:					List[str]=4096      ,       lowercase_						:					Optional[Any]=16      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					List[Any]=0.0      ,       lowercase_						:					Optional[int]=True      ,       lowercase_						:					Tuple=True      ,       lowercase_						:					Dict="gelu_new"      ,       lowercase_						:					List[str]=1024      ,       lowercase_						:					Dict=0.1      ,       lowercase_						:					Union[str, Any]=0.0      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					str=0.02      ,       lowercase_						:					List[str]=2      ,       lowercase_						:					Any=0.0      ,       lowercase_						:					int=True      ,       lowercase_						:					Union[str, Any]=0      ,       lowercase_						:					List[str]=2      ,       lowercase_						:					Optional[Any]=1      ,       lowercase_						:					Any="block_sparse"      ,       lowercase_						:					str=64      ,       lowercase_						:					Dict=3      ,       lowercase_						:					List[str]=False      ,       **lowercase_						:					List[str]      ,       ) ->						str:
             """simple docstring"""




             _lowerCamelCase	:		Optional[int]          =vocab_size
             _lowerCamelCase	:		Tuple          =max_position_embeddings
             _lowerCamelCase	:		Any          =d_model
             _lowerCamelCase	:		Optional[Any]          =encoder_ffn_dim
             _lowerCamelCase	:		Union[str, Any]          =encoder_layers
             _lowerCamelCase	:		int          =encoder_attention_heads
             _lowerCamelCase	:		Optional[Any]          =decoder_ffn_dim
             _lowerCamelCase	:		int          =decoder_layers
             _lowerCamelCase	:		str          =decoder_attention_heads
             _lowerCamelCase	:		Union[str, Any]          =dropout
             _lowerCamelCase	:		str          =attention_dropout
             _lowerCamelCase	:		int          =activation_dropout
             _lowerCamelCase	:		Union[str, Any]          =activation_function
             _lowerCamelCase	:		Optional[Any]          =init_std
             _lowerCamelCase	:		Optional[Any]          =encoder_layerdrop
             _lowerCamelCase	:		int          =decoder_layerdrop
             _lowerCamelCase	:		Optional[int]          =classifier_dropout
             _lowerCamelCase	:		int          =use_cache
             _lowerCamelCase	:		Dict          =encoder_layers
             _lowerCamelCase	:		List[str]          =scale_embedding  # scale factor will be sqrt(d_model) if True

             # extra config
             _lowerCamelCase	:		Tuple          =attention_type
             _lowerCamelCase	:		str          =block_size
             _lowerCamelCase	:		Optional[int]          =num_random_blocks
             _lowerCamelCase	:		Optional[int]          =use_bias

             super().__init__(
                 pad_token_id=lowercase_      ,       bos_token_id=lowercase_      ,       eos_token_id=lowercase_      ,       is_encoder_decoder=lowercase_      ,       decoder_start_token_id=lowercase_      ,       **lowercase_      ,       )




class A  (       UpperCamelCase_					):






      @property
      def 	lowerCamelCase	( self						:					Dict			) ->						Mapping[str, Mapping[int, str]]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Dict          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                        ]			)

                    if self.use_past:
                           _lowerCamelCase	:		str          ={0: 'batch'}
                           _lowerCamelCase	:		Optional[int]          ={0: 'batch', 1: 'past_decoder_sequence + sequence'}
                    else:
                           _lowerCamelCase	:		Optional[Any]          ={0: 'batch', 1: 'decoder_sequence'}
                           _lowerCamelCase	:		List[Any]          ={0: 'batch', 1: 'decoder_sequence'}

                    if self.use_past:
                           self.fill_with_past_key_values_(lowercase_      ,       direction='inputs'			)
             elif self.task == "causal-lm":
                    # TODO: figure this case out.
                    _lowerCamelCase	:		Union[str, Any]          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                        ]			)
                    if self.use_past:
                           _lowerCamelCase     ,					_lowerCamelCase	:		Union[str, Any]          =self.num_layers
                           for i in range(lowercase_			):
                                  _lowerCamelCase	:		Dict          ={0: 'batch', 2: 'past_sequence + sequence'}
                                  _lowerCamelCase	:		List[str]          ={0: 'batch', 2: 'past_sequence + sequence'}
             else:
                    _lowerCamelCase	:		Optional[int]          =OrderedDict(
                        [
                            ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                            ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                            ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
                            ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
                        ]			)

             return common_inputs






      @property
      def 	lowerCamelCase	( self						:					Dict			) ->						Mapping[str, Mapping[int, str]]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Dict          =super().outputs
             else:
                    _lowerCamelCase	:		Optional[Any]          =super(lowercase_      ,       self			).outputs
                    if self.use_past:
                           _lowerCamelCase     ,					_lowerCamelCase	:		str          =self.num_layers
                           for i in range(lowercase_			):
                                  _lowerCamelCase	:		Tuple          ={0: 'batch', 2: 'past_sequence + sequence'}
                                  _lowerCamelCase	:		List[Any]          ={0: 'batch', 2: 'past_sequence + sequence'}
             return common_outputs






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		int          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)

             # Generate decoder inputs
             _lowerCamelCase	:		str          =seq_length if not self.use_past else 1
             _lowerCamelCase	:		List[Any]          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)
             _lowerCamelCase	:		List[Any]          ={F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
             _lowerCamelCase	:		Union[str, Any]          =dict(**lowercase_      ,       **lowercase_			)

             if self.use_past:
                    if not is_torch_available():
                           raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'			)
                    else:
                           import torch
                    _lowerCamelCase     ,					_lowerCamelCase	:		int          =common_inputs['input_ids'].shape
                    _lowerCamelCase	:		List[Any]          =common_inputs['decoder_input_ids'].shape[1]
                    _lowerCamelCase     ,					_lowerCamelCase	:		Union[str, Any]          =self.num_attention_heads
                    _lowerCamelCase	:		Dict          =(
                        batch,
                        num_encoder_attention_heads,
                        encoder_seq_length,
                        self._config.hidden_size // num_encoder_attention_heads,
                    )
                    _lowerCamelCase	:		List[Any]          =decoder_seq_length + 3
                    _lowerCamelCase	:		int          =(
                        batch,
                        num_decoder_attention_heads,
                        decoder_past_length,
                        self._config.hidden_size // num_decoder_attention_heads,
                    )

                    _lowerCamelCase	:		Union[str, Any]          =torch.cat(
                        [common_inputs['decoder_attention_mask'], torch.ones(lowercase_      ,       lowercase_			)]      ,       dim=1			)

                    _lowerCamelCase	:		Optional[Any]          =[]
                    # If the number of encoder and decoder layers are present in the model configuration, both are considered
                    _lowerCamelCase     ,					_lowerCamelCase	:		Tuple          =self.num_layers
                    _lowerCamelCase	:		Optional[int]          =min(lowercase_      ,       lowercase_			)
                    _lowerCamelCase	:		int          =max(lowercase_      ,       lowercase_			) - min_num_layers
                    _lowerCamelCase	:		str          ='encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

                    for _ in range(lowercase_			):
                           common_inputs["past_key_values"].append(
                               (
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                                   torch.zeros(lowercase_			),
                               )			)
                    # TODO: test this.
                    _lowerCamelCase	:		List[Any]          =encoder_shape if remaining_side_name == 'encoder' else decoder_shape
                    for _ in range(lowercase_      ,       lowercase_			):
                           common_inputs["past_key_values"].append((torch.zeros(lowercase_			), torch.zeros(lowercase_			))			)
             return common_inputs






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		str          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)

             if self.use_past:
                    if not is_torch_available():
                           raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'			)
                    else:
                           import torch
                    _lowerCamelCase     ,					_lowerCamelCase	:		Optional[int]          =common_inputs['input_ids'].shape
                    # Not using the same length for past_key_values
                    _lowerCamelCase	:		Union[str, Any]          =seqlen + 2
                    _lowerCamelCase     ,					_lowerCamelCase	:		List[Any]          =self.num_layers
                    _lowerCamelCase     ,					_lowerCamelCase	:		Any          =self.num_attention_heads
                    _lowerCamelCase	:		List[str]          =(
                        batch,
                        num_encoder_attention_heads,
                        past_key_values_length,
                        self._config.hidden_size // num_encoder_attention_heads,
                    )

                    _lowerCamelCase	:		Tuple          =common_inputs['attention_mask'].dtype
                    _lowerCamelCase	:		Tuple          =torch.cat(
                        [common_inputs['attention_mask'], torch.ones(lowercase_      ,       lowercase_      ,       dtype=lowercase_			)]      ,       dim=1			)
                    _lowerCamelCase	:		Dict          =[
                        (torch.zeros(lowercase_			), torch.zeros(lowercase_			)) for _ in range(lowercase_			)
                    ]
             return common_inputs






      def 	lowerCamelCase	( self						:					List[str]      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[Any]          =compute_effective_axis_dimension(
                 lowercase_      ,       fixed_dimension=OnnxConfig.default_fixed_batch      ,       num_token_to_add=0			)

             # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
             _lowerCamelCase	:		Tuple          =tokenizer.num_special_tokens_to_add(lowercase_			)
             _lowerCamelCase	:		Any          =compute_effective_axis_dimension(
                 lowercase_      ,       fixed_dimension=OnnxConfig.default_fixed_sequence      ,       num_token_to_add=lowercase_			)

             # Generate dummy inputs according to compute batch and sequence
             _lowerCamelCase	:		Any          =[' '.join([tokenizer.unk_token]			) * seq_length] * batch_size
             _lowerCamelCase	:		int          =dict(tokenizer(lowercase_      ,       return_tensors=lowercase_			)			)
             return common_inputs






      def 	lowerCamelCase	( self						:					Any      ,       lowercase_						:					PreTrainedTokenizer      ,       lowercase_						:					int = -1      ,       lowercase_						:					int = -1      ,       lowercase_						:					bool = False      ,       lowercase_						:					Optional[TensorType] = None      ,       ) ->						Mapping[str, Any]:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Union[str, Any]          =self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)

             elif self.task == "causal-lm":
                    _lowerCamelCase	:		List[str]          =self._generate_dummy_inputs_for_causal_lm(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)
             else:
                    _lowerCamelCase	:		List[Any]          =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                        lowercase_      ,       batch_size=lowercase_      ,       seq_length=lowercase_      ,       is_pair=lowercase_      ,       framework=lowercase_			)

             return common_inputs






      def 	lowerCamelCase	( self						:					List[Any]      ,       lowercase_						:					Union[str, Any]      ,       lowercase_						:					Optional[int]      ,       lowercase_						:					Dict      ,       lowercase_						:					Tuple			) ->						Dict:
             """simple docstring"""




             if self.task in ["default", "seq2seq-lm"]:
                    _lowerCamelCase	:		Union[str, Any]          =super()._flatten_past_key_values_(lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)
             else:
                    _lowerCamelCase	:		List[str]          =super(lowercase_      ,       self			)._flatten_past_key_values_(
                        lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_			)


