

import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
       from .tokenization_llama import LlamaTokenizer
else:
       lowerCamelCase									= None

lowerCamelCase									= logging.get_logger(__name__)
lowerCamelCase									= {'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase					,	lowerCamelCase									= '''[INST]''', '''[/INST]'''
lowerCamelCase					,	lowerCamelCase									= '''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
lowerCamelCase									= '''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''







class 						__magic_name__      (							lowerCamelCase__							):

   '''simple docstring'''




   lowerCamelCase__			:     Optional[int]    =       VOCAB_FILES_NAMES
   lowerCamelCase__			:     List[str]    =       LlamaTokenizer
   lowerCamelCase__			:     Any    =       'left'
   lowerCamelCase__			:     List[Any]    =       ['input_ids', 'attention_mask']



   def __init__(    self,   lowercase_=None,   lowercase_=None,   lowercase_=False,   lowercase_="<unk>",   lowercase_="<s>",   lowercase_="</s>",   lowercase_=True,   lowercase_=False,   **lowercase_,   )							->   int:






     """simple docstring"""






     super().__init__(
         vocab_file=lowercase_,   tokenizer_file=lowercase_,   clean_up_tokenization_spaces=lowercase_,   unk_token=lowercase_,   bos_token=lowercase_,   eos_token=lowercase_,   **lowercase_,   )
     a__  =add_bos_token
     a__  =add_eos_token
     self.update_post_processor()

     a__  =vocab_file
     a__  =False if not self.vocab_file else True



   def 			_UpperCAmelCase			(    self	)							->   List[Any]:






     """simple docstring"""






     a__  =self.bos_token
     a__  =self.bos_token_id

     a__  =self.eos_token
     a__  =self.eos_token_id

     a__  =F"""{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}"""
     a__  =F"""{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}"""

     a__  =[]
     if self.add_bos_token:
       special_tokens.append((bos, bos_token_id)	)
     if self.add_eos_token:
       special_tokens.append((eos, eos_token_id)	)
     a__  =processors.TemplateProcessing(
         single=lowercase_,   pair=lowercase_,   special_tokens=lowercase_	)



   @property
   def 			_UpperCAmelCase			(    self	)							->   Optional[int]:






     """simple docstring"""






     return self._add_eos_token



   @property
   def 			_UpperCAmelCase			(    self	)							->   Any:






     """simple docstring"""






     return self._add_bos_token



   @add_eos_token.setter
   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Union[str, Any]:






     """simple docstring"""






     a__  =value
     self.update_post_processor()



   @add_bos_token.setter
   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Union[str, Any]:






     """simple docstring"""






     a__  =value
     self.update_post_processor()



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = None	)							->   Tuple[str]:






     """simple docstring"""






     if not self.can_save_slow_tokenizer:
       raise ValueError(
           '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
           '''tokenizer.'''	)

     if not os.path.isdir(lowercase_	):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
       return
     a__  =os.path.join(
         lowercase_,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

     if os.path.abspath(self.vocab_file	) != os.path.abspath(lowercase_	):
       copyfile(self.vocab_file,   lowercase_	)

     return (out_vocab_file,)



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Any:






     """simple docstring"""






     a__  =list(conversation.iter_texts()	)
     if not all([is_user for is_user, msg in dialogue[::2]]	) or not all(
         [not is_user for is_user, msg in dialogue[1::2]]	):
       raise ValueError(
           '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''	)

     a__  =[]
     if len(conversation.past_user_inputs	) > 0:
       if not conversation.past_user_inputs[0].startswith(lowercase_	) or E_SYS not in conversation.past_user_inputs[0]:
         a__  =(
             B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
         )
     elif not dialogue[0][1].startswith(lowercase_	) or E_SYS not in dialogue[0][1]:
       a__  =(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

     dialog_tokens += sum(
         [
             [self.bos_token_id]
             + self.encode(
                 F"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """,   add_special_tokens=lowercase_	)
             + [self.eos_token_id]
             for prompt, answer in zip(dialogue[::2],   dialogue[1::2]	)
         ],   [],   )
     if not (dialogue[-1][0]):
       raise ValueError(F"""Last message must be from user, got {dialogue[-1]["role"]}"""	)
     dialog_tokens += [self.bos_token_id] + self.encode(
         F"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}""",   add_special_tokens=lowercase_	)
     return dialog_tokens



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


lowerCamelCase									= logging.get_logger(__name__)

lowerCamelCase									= {
    '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json''',
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}







class 						__magic_name__      (							lowerCamelCase__							):

   '''simple docstring'''




   lowerCamelCase__			:     List[Any]    =       'blenderbot'
   lowerCamelCase__			:     int    =       ['past_key_values']
   lowerCamelCase__			:     Optional[Any]    =       {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}



   def __init__(    self,   lowercase_=8008,   lowercase_=128,   lowercase_=2,   lowercase_=10240,   lowercase_=32,   lowercase_=24,   lowercase_=10240,   lowercase_=32,   lowercase_=0.0,   lowercase_=0.0,   lowercase_=True,   lowercase_=True,   lowercase_="gelu",   lowercase_=2560,   lowercase_=0.1,   lowercase_=0.0,   lowercase_=0.0,   lowercase_=0.02,   lowercase_=1,   lowercase_=False,   lowercase_=0,   lowercase_=1,   lowercase_=2,   lowercase_=3,   lowercase_=2,   **lowercase_,   )							->   Any:






     """simple docstring"""






     a__  =vocab_size
     a__  =max_position_embeddings
     a__  =d_model
     a__  =encoder_ffn_dim
     a__  =encoder_layers
     a__  =encoder_attention_heads
     a__  =decoder_ffn_dim
     a__  =decoder_layers
     a__  =decoder_attention_heads
     a__  =dropout
     a__  =attention_dropout
     a__  =activation_dropout
     a__  =activation_function
     a__  =init_std
     a__  =encoder_layerdrop
     a__  =decoder_layerdrop
     a__  =use_cache
     a__  =encoder_layers
     a__  =scale_embedding  # scale factor will be sqrt(d_model) if True

     super().__init__(
         pad_token_id=lowercase_,   bos_token_id=lowercase_,   eos_token_id=lowercase_,   is_encoder_decoder=lowercase_,   decoder_start_token_id=lowercase_,   encoder_no_repeat_ngram_size=lowercase_,   forced_eos_token_id=lowercase_,   **lowercase_,   )







class 						__magic_name__      (							lowerCamelCase__							):

   '''simple docstring'''




   @property
   def 			_UpperCAmelCase			(    self	)							->   Mapping[str, Mapping[int, str]]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]	)
       if self.use_past:
         a__  ={0: '''batch'''}
         a__  ={0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
         a__  ={0: '''batch''', 1: '''decoder_sequence'''}
         a__  ={0: '''batch''', 1: '''decoder_sequence'''}
       if self.use_past:
         self.fill_with_past_key_values_(lowercase_,   direction='''inputs'''	)
     elif self.task == "causal-lm":
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]	)
       if self.use_past:
         a__,				a__  =self.num_layers
         for i in range(lowercase_	):
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
     else:
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
               ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
           ]	)

     return common_inputs



   @property
   # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
   def 			_UpperCAmelCase			(    self	)							->   Mapping[str, Mapping[int, str]]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =super().outputs
     else:
       a__  =super(lowercase_,   self	).outputs
       if self.use_past:
         a__,				a__  =self.num_layers
         for i in range(lowercase_	):
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
     return common_outputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     # Generate decoder inputs
     a__  =seq_length if not self.use_past else 1
     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     a__  ={F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
     a__  =dict(**lowercase_,   **lowercase_	)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
       else:
         import torch
       a__,				a__  =common_inputs['''input_ids'''].shape
       a__  =common_inputs['''decoder_input_ids'''].shape[1]
       a__,				a__  =self.num_attention_heads
       a__  =(
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       a__  =decoder_seq_length
       a__  =(
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )
       a__  =torch.cat(
           [common_inputs['''decoder_attention_mask'''], torch.ones(lowercase_,   lowercase_	)],   dim=1	)
       a__  =[]
       a__,				a__  =self.num_layers

       for _ in range(lowercase_	):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
             )	)
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
       else:
         import torch
       a__,				a__  =common_inputs['''input_ids'''].shape
       a__  =seqlen
       a__,				a__  =self.num_layers
       a__,				a__  =self.num_attention_heads
       a__  =(
           batch,
           num_encoder_attention_heads,
           past_key_values_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       a__  =common_inputs['''attention_mask'''].dtype
       a__  =torch.cat(
           [common_inputs['''attention_mask'''], torch.ones(lowercase_,   lowercase_,   dtype=lowercase_	)],   dim=1	)
       a__  =[
           (torch.zeros(lowercase_	), torch.zeros(lowercase_	)) for _ in range(lowercase_	)
       ]
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =compute_effective_axis_dimension(
         lowercase_,   fixed_dimension=OnnxConfig.default_fixed_batch,   num_token_to_add=0	)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     a__  =tokenizer.num_special_tokens_to_add(lowercase_	)
     a__  =compute_effective_axis_dimension(
         lowercase_,   fixed_dimension=OnnxConfig.default_fixed_sequence,   num_token_to_add=lowercase_	)

     # Generate dummy inputs according to compute batch and sequence
     a__  =[''' '''.join([tokenizer.unk_token]	) * seq_length] * batch_size
     a__  =dict(tokenizer(lowercase_,   return_tensors=lowercase_	)	)
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =self._generate_dummy_inputs_for_default_and_seqaseq_lm(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)

     elif self.task == "causal-lm":
       a__  =self._generate_dummy_inputs_for_causal_lm(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)
     else:
       a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)

     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)							->   Optional[int]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =super()._flatten_past_key_values_(lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     else:
       a__  =super(lowercase_,   self	)._flatten_past_key_values_(
           lowercase_,   lowercase_,   lowercase_,   lowercase_	)



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_	)							->   Tuple:






     """simple docstring"""






     if direction not in ["inputs", "outputs"]:
       raise ValueError(F"""direction must either be \"inputs\" or \"outputs\", but {direction} was given"""	)

     a__  ='''past_key_values''' if direction == '''inputs''' else '''present'''
     a__,				a__  =self.num_layers

     a__  ='''past_encoder_sequence'''
     a__  ='''past_decoder_sequence''' if direction == '''inputs''' else '''past_decoder_sequence + sequence'''

     for i in range(lowercase_	):
       a__  ={0: '''batch''', 2: decoder_sequence}
       a__  ={0: '''batch''', 2: decoder_sequence}
       a__  ={0: '''batch''', 2: encoder_sequence}
       a__  ={0: '''batch''', 2: encoder_sequence}
