



'''simple docstring'''





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

require_version("tokenizers>=0.13.3")

if is_sentencepiece_available():
  from .tokenization_llama import LlamaTokenizer
else:
  lowercase_			=  None

lowercase_			=  logging.get_logger(__name__)
lowercase_			=  {"vocab_file": "tokenizer.model", "tokenizer_file": "tokenizer.json"}

lowercase_	, lowercase_			=  "[INST]", "[/INST]"
lowercase_	, lowercase_			=  "<<SYS>>\n", "\n<</SYS>>\n\n"

# fmt: off
lowercase_			=  "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don't know the answer to a question, please don't share false information."






class __A      (  A						):



 '''simple docstring'''







 __lowerCamelCase   :	List[Any]										=  VOCAB_FILES_NAMES
 __lowerCamelCase   :	Optional[int]										=  LlamaTokenizer
 __lowerCamelCase   :	Union[str, Any]										=  'left'
 __lowerCamelCase   :	str										=  ['input_ids', 'attention_mask']



 def __init__(self			,       A=None			,       A=None			,       A=False			,       A="<unk>"			,       A="<s>"			,       A="</s>"			,       A=True			,       A=False			,       **A			,       )   ->					Dict:


      """simple docstring"""



      super().__init__(
          vocab_file=A			,       tokenizer_file=A			,       clean_up_tokenization_spaces=A			,       unk_token=A			,       bos_token=A			,       eos_token=A			,       **A			,       )
      _a     =  add_bos_token
      _a     =  add_eos_token
      self.update_post_processor()

      _a     =  vocab_file
      _a     =  False if not self.vocab_file else True



 def  a__  (self	)   ->					str:


      """simple docstring"""



      _a     =  self.bos_token
      _a     =  self.bos_token_id

      _a     =  self.eos_token
      _a     =  self.eos_token_id

      _a     =  f'''{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}'''
      _a     =  f'''{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}'''

      _a     =  []
      if self.add_bos_token:
           special_tokens.append((bos, bos_token_id)	)
      if self.add_eos_token:
           special_tokens.append((eos, eos_token_id)	)
      _a     =  processors.TemplateProcessing(
          single=A			,       pair=A			,       special_tokens=A	)



 @property
 def  a__  (self	)   ->					Any:


      """simple docstring"""



      return self._add_eos_token



 @property
 def  a__  (self	)   ->					str:


      """simple docstring"""



      return self._add_bos_token



 @add_eos_token.setter
 def  a__  (self			,       A	)   ->					Any:


      """simple docstring"""



      _a     =  value
      self.update_post_processor()



 @add_bos_token.setter
 def  a__  (self			,       A	)   ->					Optional[int]:


      """simple docstring"""



      _a     =  value
      self.update_post_processor()



 def  a__  (self			,       A			,       A = None	)   ->					Tuple[str]:


      """simple docstring"""



      if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.'''	)

      if not os.path.isdir(A	):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
           return
      _a     =  os.path.join(
          A			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(A	):
           copyfile(self.vocab_file			,       A	)

      return (out_vocab_file,)



 def  a__  (self			,       A	)   ->					Tuple:


      """simple docstring"""



      _a     =  list(conversation.iter_texts()	)
      if not all([is_user for is_user, msg in dialogue[::2]]	) or not all(
          [not is_user for is_user, msg in dialogue[1::2]]	):
           raise ValueError(
               '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''	)

      _a     =  []
      if len(conversation.past_user_inputs	) > 0:
           if not conversation.past_user_inputs[0].startswith(A	) or E_SYS not in conversation.past_user_inputs[0]:
                _a     =  (
                    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                )
      elif not dialogue[0][1].startswith(A	) or E_SYS not in dialogue[0][1]:
           _a     =  (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

      dialog_tokens += sum(
          [
              [self.bos_token_id]
              + self.encode(
                  f'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''			,       add_special_tokens=A	)
              + [self.eos_token_id]
              for prompt, answer in zip(dialogue[::2]			,       dialogue[1::2]	)
          ]			,       []			,       )
      if not (dialogue[-1][0]):
           raise ValueError(f'''Last message must be from user, got {dialogue[-1]['role']}'''	)
      dialog_tokens += [self.bos_token_id] + self.encode(
          f'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''			,       add_special_tokens=A	)
      return dialog_tokens








'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  lowercase_			=  None


lowercase_			=  logging.get_logger(__name__)

lowercase_			=  {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowercase_			=  {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

lowercase_			=  {
    "facebook/mbart-large-50-one-to-many-mmt": 1_024,
}

# fmt: off
lowercase_			=  ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]






class __A      (  A						):



 '''simple docstring'''







 __lowerCamelCase   :	List[str]										=	VOCAB_FILES_NAMES
 __lowerCamelCase   :	int										=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCamelCase   :	Optional[int]										=	PRETRAINED_VOCAB_FILES_MAP
 __lowerCamelCase   :	Any										=	['input_ids', 'attention_mask']
 __lowerCamelCase   :	Optional[Any]										=	MBartaaTokenizer

 __lowerCamelCase   :	List[int]										=	[]
 __lowerCamelCase   :	List[int]										=	[]



 def __init__(self			,       A=None			,       A=None			,       A=None			,       A=None			,       A="</s>"			,       A="</s>"			,       A="<s>"			,       A="<unk>"			,       A="<pad>"			,       A="<mask>"			,       **A			,       )   ->					List[str]:


      """simple docstring"""



      _a     =  AddedToken(A			,       lstrip=A			,       rstrip=A	) if isinstance(A			,       A	) else mask_token

      _a     =  kwargs.get('''additional_special_tokens'''			,       []	)
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          A			,       src_lang=A			,       tgt_lang=A			,       tokenizer_file=A			,       eos_token=A			,       sep_token=A			,       cls_token=A			,       unk_token=A			,       pad_token=A			,       mask_token=A			,       **A			,       )

      _a     =  vocab_file
      _a     =  False if not self.vocab_file else True

      _a     =  {
          lang_code: self.convert_tokens_to_ids(A	) for lang_code in FAIRSEQ_LANGUAGE_CODES
      }

      _a     =  src_lang if src_lang is not None else '''en_XX'''
      _a     =  tgt_lang
      _a     =  self.lang_code_to_id[self._src_lang]
      self.set_src_lang_special_tokens(self._src_lang	)



 @property
 def  a__  (self	)   ->					str:


      """simple docstring"""



      return self._src_lang



 @src_lang.setter
 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  new_src_lang
      self.set_src_lang_special_tokens(self._src_lang	)



 def  a__  (self			,       A			,       A = None	)   ->					List[int]:


      """simple docstring"""



      if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



 def  a__  (self			,       A			,       A = "en_XX"			,       A = None			,       A = "ro_RO"			,       **A			,       )   ->					BatchEncoding:


      """simple docstring"""



      _a     =  src_lang
      _a     =  tgt_lang
      return super().prepare_seqaseq_batch(A			,       A			,       **A	)



 def  a__  (self	)   ->					Union[str, Any]:


      """simple docstring"""



      return self.set_src_lang_special_tokens(self.src_lang	)



 def  a__  (self	)   ->					Union[str, Any]:


      """simple docstring"""



      return self.set_tgt_lang_special_tokens(self.tgt_lang	)



 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  [self.cur_lang_code_id]
      _a     =  [self.eos_token_id]

      _a     =  self.convert_ids_to_tokens(self.prefix_tokens	)
      _a     =  self.convert_ids_to_tokens(self.suffix_tokens	)

      _a     =  processors.TemplateProcessing(
          single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str			,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str			,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,       self.prefix_tokens + self.suffix_tokens	)	)			,       )



 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  [self.cur_lang_code_id]
      _a     =  [self.eos_token_id]

      _a     =  self.convert_ids_to_tokens(self.prefix_tokens	)
      _a     =  self.convert_ids_to_tokens(self.suffix_tokens	)

      _a     =  processors.TemplateProcessing(
          single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str			,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str			,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,       self.prefix_tokens + self.suffix_tokens	)	)			,       )



 def  a__  (self			,       A			,       A			,       A			,       A			,       **A	)   ->					Optional[int]:


      """simple docstring"""



      if src_lang is None or tgt_lang is None:
           raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''	)
      _a     =  src_lang
      _a     =  self(A			,       add_special_tokens=A			,       return_tensors=A			,       **A	)
      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  tgt_lang_id
      return inputs



 def  a__  (self			,       A			,       A = None	)   ->					Tuple[str]:


      """simple docstring"""



      if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.'''	)

      if not os.path.isdir(A	):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
           return
      _a     =  os.path.join(
          A			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(A	):
           copyfile(self.vocab_file			,       A	)

      return (out_vocab_file,)



