






"""simple docstring"""





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
       from .tokenization_llama import LlamaTokenizer
else:
       _SCREAMING_SNAKE_CASE :    str    =						None

_SCREAMING_SNAKE_CASE :    int    =						logging.get_logger(__name__)
_SCREAMING_SNAKE_CASE :    Optional[int]    =						{'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE   ,							_SCREAMING_SNAKE_CASE :    int    =						'''[INST]''', '''[/INST]'''
_SCREAMING_SNAKE_CASE   ,							_SCREAMING_SNAKE_CASE :    int    =						'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
_SCREAMING_SNAKE_CASE :    int    =						'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'''






class 					a						( __lowerCamelCase		):
    SCREAMING_SNAKE_CASE						:					Dict						=			VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						:					Tuple						=			LlamaTokenizer
    SCREAMING_SNAKE_CASE						:					int						=			"left"
    SCREAMING_SNAKE_CASE						:					List[Any]						=			["input_ids", "attention_mask"]






    def __init__(	self					:							Any	,						__SCREAMING_SNAKE_CASE					:							str=None	,						__SCREAMING_SNAKE_CASE					:							str=None	,						__SCREAMING_SNAKE_CASE					:							Any=False	,						__SCREAMING_SNAKE_CASE					:							Optional[int]="<unk>"	,						__SCREAMING_SNAKE_CASE					:							int="<s>"	,						__SCREAMING_SNAKE_CASE					:							int="</s>"	,						__SCREAMING_SNAKE_CASE					:							str=True	,						__SCREAMING_SNAKE_CASE					:							Union[str, Any]=False	,						**__SCREAMING_SNAKE_CASE					:							Dict	,						)					->		List[Any]:
      super().__init__(
          vocab_file=_UpperCamelCase	,						tokenizer_file=_UpperCamelCase	,						clean_up_tokenization_spaces=_UpperCamelCase	,						unk_token=_UpperCamelCase	,						bos_token=_UpperCamelCase	,						eos_token=_UpperCamelCase	,						**_UpperCamelCase	,						)
      lowerCamelCase_   					=						add_bos_token
      lowerCamelCase_   					=						add_eos_token
      self.update_post_processor()

      lowerCamelCase_   					=						vocab_file
      lowerCamelCase_   					=						False if not self.vocab_file else True






    def        UpperCamelCase	(	self					:							Union[str, Any] )					->		Optional[Any]:
      lowerCamelCase_   					=						self.bos_token
      lowerCamelCase_   					=						self.bos_token_id

      lowerCamelCase_   					=						self.eos_token
      lowerCamelCase_   					=						self.eos_token_id

      lowerCamelCase_   					=						F'''{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'''
      lowerCamelCase_   					=						F'''{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'''

      lowerCamelCase_   					=						[]
      if self.add_bos_token:
        special_tokens.append((bos, bos_token_id) )
      if self.add_eos_token:
        special_tokens.append((eos, eos_token_id) )
      lowerCamelCase_   					=						processors.TemplateProcessing(
          single=_UpperCamelCase	,						pair=_UpperCamelCase	,						special_tokens=_UpperCamelCase )






    @property
    def        UpperCamelCase	(	self					:							Optional[Any] )					->		List[str]:
      return self._add_eos_token






    @property
    def        UpperCamelCase	(	self					:							Optional[int] )					->		Any:
      return self._add_bos_token






    @add_eos_token.setter
    def        UpperCamelCase	(	self					:							List[str]	,						__SCREAMING_SNAKE_CASE					:							Optional[Any] )					->		Any:
      lowerCamelCase_   					=						value
      self.update_post_processor()






    @add_bos_token.setter
    def        UpperCamelCase	(	self					:							Union[str, Any]	,						__SCREAMING_SNAKE_CASE					:							Tuple )					->		str:
      lowerCamelCase_   					=						value
      self.update_post_processor()






    def        UpperCamelCase	(	self					:							Optional[Any]	,						__SCREAMING_SNAKE_CASE					:							str	,						__SCREAMING_SNAKE_CASE					:							Optional[str] = None )					->		Tuple[str]:





      if not self.can_save_slow_tokenizer:
        raise ValueError(
            'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
            'tokenizer.' )

      if not os.path.isdir(_UpperCamelCase ):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
        return
      lowerCamelCase_   					=						os.path.join(
          _UpperCamelCase	,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

      if os.path.abspath(self.vocab_file ) != os.path.abspath(_UpperCamelCase ):
        copyfile(self.vocab_file	,						_UpperCamelCase )

      return (out_vocab_file,)






    def        UpperCamelCase	(	self					:							Optional[Any]	,						__SCREAMING_SNAKE_CASE					:							"Conversation" )					->		Union[str, Any]:
      lowerCamelCase_   					=						list(conversation.iter_texts() )
      if not all([is_user for is_user, msg in dialogue[::2]] ) or not all(
          [not is_user for is_user, msg in dialogue[1::2]] ):
        raise ValueError(
            'The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)' )

      lowerCamelCase_   					=						[]
      if len(conversation.past_user_inputs ) > 0:
        if not conversation.past_user_inputs[0].startswith(_UpperCamelCase ) or E_SYS not in conversation.past_user_inputs[0]:
          lowerCamelCase_   					=						(
              B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
          )
      elif not dialogue[0][1].startswith(_UpperCamelCase ) or E_SYS not in dialogue[0][1]:
        lowerCamelCase_   					=						(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

      dialog_tokens += sum(
          [
              [self.bos_token_id]
              + self.encode(
                  F'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''	,						add_special_tokens=_UpperCamelCase )
              + [self.eos_token_id]
              for prompt, answer in zip(dialogue[::2]	,						dialogue[1::2] )
          ]	,						[]	,						)
      if not (dialogue[-1][0]):
        raise ValueError(F'''Last message must be from user, got {dialogue[-1]["role"]}''' )
      dialog_tokens += [self.bos_token_id] + self.encode(
          F'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''	,						add_special_tokens=_UpperCamelCase )
      return dialog_tokens












"""simple docstring"""





import numpy as np







def 					lowerCamelCase__	(							_lowerCamelCase	:		Dict		,    _lowerCamelCase	:		Any		,    _lowerCamelCase	:		Tuple			)				->    str:
  lowerCamelCase_   					=						n_draws * class_counts / class_counts.sum()
  # floored means we don't overshoot n_samples, but probably undershoot
  lowerCamelCase_   					=						np.floor(_lowerCamelCase			)
  # we add samples according to how much "left over" probability
  # they had, until we arrive at n_samples
  lowerCamelCase_   					=						int(n_draws - floored.sum()			)
  if need_to_add > 0:
    lowerCamelCase_   					=						continuous - floored
    lowerCamelCase_   					=						np.sort(np.unique(_lowerCamelCase			)			)[::-1]
    # add according to remainder, but break ties
    # randomly to avoid biases
    for value in values:
      ((lowerCamelCase_)					,     )   					=						np.where(remainder == value			)
      # if we need_to_add less than what's in inds
      # we draw randomly from them.
      # if we need to add more, we add them all and
      # go to the next value
      lowerCamelCase_   					=						min(len(_lowerCamelCase			)		,    _lowerCamelCase			)
      lowerCamelCase_   					=						rng.choice(_lowerCamelCase		,    size=_lowerCamelCase		,    replace=_lowerCamelCase			)
      floored[inds] += 1
      need_to_add -= add_now
      if need_to_add == 0:
        break
  return floored.astype(np.intaa			)







def 					lowerCamelCase__	(							_lowerCamelCase	:		Any		,    _lowerCamelCase	:		int		,    _lowerCamelCase	:		Optional[Any]		,    _lowerCamelCase	:		List[str]		,    _lowerCamelCase	:		Optional[int]=10			)				->    Any:
  lowerCamelCase_					,     lowerCamelCase_   					=						np.unique(_lowerCamelCase		,    return_inverse=_lowerCamelCase			)
  lowerCamelCase_   					=						classes.shape[0]
  lowerCamelCase_   					=						np.bincount(_lowerCamelCase			)
  if np.min(_lowerCamelCase			) < 2:
    raise ValueError('Minimum class count error'			)
  if n_train < n_classes:
    raise ValueError(
        'The train_size = %d should be greater or ' 'equal to the number of classes = %d' % (n_train, n_classes)			)
  if n_test < n_classes:
    raise ValueError(
        'The test_size = %d should be greater or ' 'equal to the number of classes = %d' % (n_test, n_classes)			)
  lowerCamelCase_   					=						np.split(np.argsort(_lowerCamelCase		,    kind='mergesort'			)		,    np.cumsum(_lowerCamelCase			)[:-1]			)
  for _ in range(_lowerCamelCase			):
    lowerCamelCase_   					=						approximate_mode(_lowerCamelCase		,    _lowerCamelCase		,    _lowerCamelCase			)
    lowerCamelCase_   					=						class_counts - n_i
    lowerCamelCase_   					=						approximate_mode(_lowerCamelCase		,    _lowerCamelCase		,    _lowerCamelCase			)

    lowerCamelCase_   					=						[]
    lowerCamelCase_   					=						[]

    for i in range(_lowerCamelCase			):
      lowerCamelCase_   					=						rng.permutation(class_counts[i]			)
      lowerCamelCase_   					=						class_indices[i].take(_lowerCamelCase		,    mode='clip'			)
      train.extend(perm_indices_class_i[: n_i[i]]			)
      test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]]			)
    lowerCamelCase_   					=						rng.permutation(_lowerCamelCase			)
    lowerCamelCase_   					=						rng.permutation(_lowerCamelCase			)

    yield train, test




