





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation

_a									=				logging.get_logger(__name__)

_a									=				{"vocab_file": "tokenizer.model"}

_a									=				{
    "vocab_file": {
        "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model",
    },
    "tokenizer_file": {
        "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json",
    },
}
_a									=				{
    "hf-internal-testing/llama-tokenizer": 2_048,
}
_a									=				"▁"

_a						,		_a									=				"[INST]", "[/INST]"
_a						,		_a									=				"<<SYS>>\n", "\n<</SYS>>\n\n"

# fmt: off
_a									=				"You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don't know the answer to a question, please don't share false information."






class  __A   (   lowerCAmelCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						VOCAB_FILES_NAMES
			lowerCAmelCase_												=						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_												=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_												=						["""input_ids""", """attention_mask"""]





			def __init__(   self	,  __lowerCAmelCase	,  __lowerCAmelCase="<unk>"	,  __lowerCAmelCase="<s>"	,  __lowerCAmelCase="</s>"	,  __lowerCAmelCase=None	,  __lowerCAmelCase = None	,  __lowerCAmelCase=True	,  __lowerCAmelCase=False	,  __lowerCAmelCase=False	,  __lowerCAmelCase=True	,  **__lowerCAmelCase	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		{} if sp_model_kwargs is None else sp_model_kwargs
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else bos_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else eos_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else unk_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else pad_token
				super().__init__(
				    bos_token=__lowerCAmelCase	,  eos_token=__lowerCAmelCase	,  unk_token=__lowerCAmelCase	,  pad_token=__lowerCAmelCase	,  add_bos_token=__lowerCAmelCase	,  add_eos_token=__lowerCAmelCase	,  sp_model_kwargs=self.sp_model_kwargs	,  clean_up_tokenization_spaces=__lowerCAmelCase	,  legacy=__lowerCAmelCase	,  **__lowerCAmelCase	,  )
				if legacy:
					logger.warning_once(
					    F'You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'
					    ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''							)
				lowerCamelCase__        =		legacy
				lowerCamelCase__        =		vocab_file
				lowerCamelCase__        =		add_bos_token
				lowerCamelCase__        =		add_eos_token
				lowerCamelCase__        =		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(__lowerCAmelCase							)





			def __getstate__(   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.__dict__.copy()
				lowerCamelCase__        =		None
				lowerCamelCase__        =		self.sp_model.serialized_model_proto()
				return state





			def __setstate__(   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		d
				lowerCamelCase__        =		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)





			@property
			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return self.sp_model.get_piece_size()





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		{self.convert_ids_to_tokens(__lowerCAmelCase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  **__lowerCAmelCase							):



				'''simple docstring'''







				if not self.legacy:
					lowerCamelCase__        =		SPIECE_UNDERLINE + text.replace(__lowerCAmelCase	,  ''' '''							)
				return super().tokenize(__lowerCAmelCase	,  **__lowerCAmelCase							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				if not self.legacy:
					lowerCamelCase__        =		text.startswith(__lowerCAmelCase							)
					if is_first:
						lowerCamelCase__        =		text[1:]

				lowerCamelCase__        =		self.sp_model.encode(__lowerCAmelCase	,  out_type=__lowerCAmelCase							)

				if not self.legacy and not is_first and not text.startswith(''' '''							) and tokens[0].startswith(__lowerCAmelCase							):
					lowerCamelCase__        =		([tokens[0][1:]] if len(tokens[0]							) > 1 else []) + tokens[1:]
				return tokens





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.piece_to_id(__lowerCAmelCase							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		self.sp_model.IdToPiece(__lowerCAmelCase							)
				return token





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		[]
				lowerCamelCase__        =		''''''
				lowerCamelCase__        =		False
				for i, token in enumerate(__lowerCAmelCase							):
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						if not prev_is_special and i != 0:
							out_string += " "
						out_string += self.sp_model.decode(__lowerCAmelCase							) + token
						lowerCamelCase__        =		True
						lowerCamelCase__        =		[]
					else:
						current_sub_tokens.append(__lowerCAmelCase							)
						lowerCamelCase__        =		False
				out_string += self.sp_model.decode(__lowerCAmelCase							)
				return out_string





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if not os.path.isdir(__lowerCAmelCase							):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
					return
				lowerCamelCase__        =		os.path.join(
				    __lowerCAmelCase	,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

				if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCAmelCase							) and os.path.isfile(self.vocab_file							):
					copyfile(self.vocab_file	,  __lowerCAmelCase							)
				elif not os.path.isfile(self.vocab_file							):
					with open(__lowerCAmelCase	,  '''wb'''							) as fi:
						lowerCamelCase__        =		self.sp_model.serialized_model_proto()
						fi.write(__lowerCAmelCase							)

				return (out_vocab_file,)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase=None							):



				'''simple docstring'''







				lowerCamelCase__        =		[self.bos_token_id] if self.add_bos_token else []
				lowerCamelCase__        =		[self.eos_token_id] if self.add_eos_token else []

				lowerCamelCase__        =		bos_token_id + token_ids_a + eos_token_id

				if token_ids_a is not None:
					lowerCamelCase__        =		output + bos_token_id + token_ids_a + eos_token_id

				return output





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase = None	,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__lowerCAmelCase	,  token_ids_a=__lowerCAmelCase	,  already_has_special_tokens=__lowerCAmelCase							)

				lowerCamelCase__        =		[1] if self.add_bos_token else []
				lowerCamelCase__        =		[1] if self.add_eos_token else []

				if token_ids_a is None:
					return bos_token_id + ([0] * len(__lowerCAmelCase							)) + eos_token_id
				return (
				    bos_token_id
				    + ([0] * len(__lowerCAmelCase							))
				    + eos_token_id
				    + bos_token_id
				    + ([0] * len(__lowerCAmelCase							))
				    + eos_token_id
				)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__        =		[self.bos_token_id] if self.add_bos_token else []
				lowerCamelCase__        =		[self.eos_token_id] if self.add_eos_token else []

				lowerCamelCase__        =		[0] * len(bos_token_id + token_ids_a + eos_token_id							)

				if token_ids_a is not None:
					output += [1] * len(bos_token_id + token_ids_a + eos_token_id							)

				return output





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		list(conversation.iter_texts()							)
				if not all([is_user for is_user, msg in dialogue[::2]]							) or not all(
				    [not is_user for is_user, msg in dialogue[1::2]]							):
					raise ValueError(
					    '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''							)

				lowerCamelCase__        =		[]
				if len(conversation.past_user_inputs							) > 0:
					if not conversation.past_user_inputs[0].startswith(__lowerCAmelCase							) or E_SYS not in conversation.past_user_inputs[0]:
						lowerCamelCase__        =		(
						    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
						)
				elif not dialogue[0][1].startswith(__lowerCAmelCase							) or E_SYS not in dialogue[0][1]:
					lowerCamelCase__        =		(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

				dialog_tokens += sum(
				    [
				        [self.bos_token_id]
				        + self.encode(
				            F'{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '	,  add_special_tokens=__lowerCAmelCase							)
				        + [self.eos_token_id]
				        for prompt, answer in zip(dialogue[::2]	,  dialogue[1::2]							)
				    ]	,  []	,  )
				if not (dialogue[-1][0]):
					raise ValueError(F'Last message must be from user, got {dialogue[-1]["role"]}'							)
				dialog_tokens += [self.bos_token_id] + self.encode(
				    F'{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'	,  add_special_tokens=__lowerCAmelCase							)
				return dialog_tokens










from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, logging, randn_tensor
from .scheduling_utils import SchedulerMixin


_a									=				logging.get_logger(__name__)  # pylint: disable=invalid-name






@dataclass
class  __A   (   lowerCAmelCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						42






class  __A   (   lowerCAmelCase      ,    lowerCAmelCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						1





			@register_to_config
			def __init__(   self	,  __lowerCAmelCase = 4_0	,  __lowerCAmelCase = 0.002	,  __lowerCAmelCase = 80.0	,  __lowerCAmelCase = 0.5	,  __lowerCAmelCase = 1.0	,  __lowerCAmelCase = 7.0	,  __lowerCAmelCase = True	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		sigma_max

				lowerCamelCase__        =		np.linspace(0	,  1	,  __lowerCAmelCase							)
				lowerCamelCase__        =		self._convert_to_karras(__lowerCAmelCase							)
				lowerCamelCase__        =		self.sigma_to_t(__lowerCAmelCase							)

				# setable values
				lowerCamelCase__        =		None
				lowerCamelCase__        =		torch.from_numpy(__lowerCAmelCase							)
				lowerCamelCase__        =		torch.from_numpy(__lowerCAmelCase							)
				lowerCamelCase__        =		False
				lowerCamelCase__        =		False





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase=None							):



				'''simple docstring'''







				if schedule_timesteps is None:
					lowerCamelCase__        =		self.timesteps

				lowerCamelCase__        =		(schedule_timesteps == timestep).nonzero()
				return indices.item()





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase							):



				'''simple docstring'''







				if isinstance(__lowerCAmelCase	,  torch.Tensor							):
					lowerCamelCase__        =		timestep.to(self.timesteps.device							)
				lowerCamelCase__        =		self.index_for_timestep(__lowerCAmelCase							)
				lowerCamelCase__        =		self.sigmas[step_idx]

				lowerCamelCase__        =		sample / ((sigma**2 + self.config.sigma_data**2) ** 0.5)

				lowerCamelCase__        =		True
				return sample





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				if not isinstance(__lowerCAmelCase	,  np.ndarray							):
					lowerCamelCase__        =		np.array(__lowerCAmelCase	,  dtype=np.floataa							)

				lowerCamelCase__        =		1_0_0_0 * 0.25 * np.log(sigmas + 1E-44							)

				return timesteps





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase = None	,  __lowerCAmelCase = None	,  __lowerCAmelCase = None	,  ):



				'''simple docstring'''







				if num_inference_steps is None and timesteps is None:
					raise ValueError('''Exactly one of `num_inference_steps` or `timesteps` must be supplied.'''							)

				if num_inference_steps is not None and timesteps is not None:
					raise ValueError('''Can only pass one of `num_inference_steps` or `timesteps`.'''							)

				# Follow DDPMScheduler custom timesteps logic
				if timesteps is not None:
					for i in range(1	,  len(__lowerCAmelCase							)							):
						if timesteps[i] >= timesteps[i - 1]:
							raise ValueError('''`timesteps` must be in descending order.'''							)

					if timesteps[0] >= self.config.num_train_timesteps:
						raise ValueError(
						    F'`timesteps` must start before `self.config.train_timesteps`:'
						    F' {self.config.num_train_timesteps}.'							)

					lowerCamelCase__        =		np.array(__lowerCAmelCase	,  dtype=np.intaa							)
					lowerCamelCase__        =		True
				else:
					if num_inference_steps > self.config.num_train_timesteps:
						raise ValueError(
						    F'`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:'
						    F' {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle'
						    F' maximal {self.config.num_train_timesteps} timesteps.'							)

					lowerCamelCase__        =		num_inference_steps

					lowerCamelCase__        =		self.config.num_train_timesteps // self.num_inference_steps
					lowerCamelCase__        =		(np.arange(0	,  __lowerCAmelCase							) * step_ratio).round()[::-1].copy().astype(np.intaa							)
					lowerCamelCase__        =		False

				# Map timesteps to Karras sigmas directly for multistep sampling
				# See https://github.com/openai/consistency_models/blob/main/cm/karras_diffusion.py#L675
				lowerCamelCase__        =		self.config.num_train_timesteps
				lowerCamelCase__        =		timesteps[::-1].copy()
				lowerCamelCase__        =		ramp / (num_train_timesteps - 1)
				lowerCamelCase__        =		self._convert_to_karras(__lowerCAmelCase							)
				lowerCamelCase__        =		self.sigma_to_t(__lowerCAmelCase							)

				lowerCamelCase__        =		np.concatenate([sigmas, [self.sigma_min]]							).astype(np.floataa							)
				lowerCamelCase__        =		torch.from_numpy(__lowerCAmelCase							).to(device=__lowerCAmelCase							)

				if str(__lowerCAmelCase							).startswith('''mps'''							):
					# mps does not support float64
					lowerCamelCase__        =		torch.from_numpy(__lowerCAmelCase							).to(__lowerCAmelCase	,  dtype=torch.floataa							)
				else:
					lowerCamelCase__        =		torch.from_numpy(__lowerCAmelCase							).to(device=__lowerCAmelCase							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		self.config.sigma_min
				lowerCamelCase__        =		self.config.sigma_max

				lowerCamelCase__        =		self.config.rho
				lowerCamelCase__        =		sigma_min ** (1 / rho)
				lowerCamelCase__        =		sigma_max ** (1 / rho)
				lowerCamelCase__        =		(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
				return sigmas





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		self.config.sigma_data

				lowerCamelCase__        =		sigma_data**2 / (sigma**2 + sigma_data**2)
				lowerCamelCase__        =		sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5
				return c_skip, c_out





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		self.config.sigma_min
				lowerCamelCase__        =		self.config.sigma_data

				lowerCamelCase__        =		sigma_data**2 / ((sigma - sigma_min) ** 2 + sigma_data**2)
				lowerCamelCase__        =		(sigma - sigma_min) * sigma_data / (sigma**2 + sigma_data**2) ** 0.5
				return c_skip, c_out





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase = None	,  __lowerCAmelCase = True	,  ):



				'''simple docstring'''







				if (
				    isinstance(__lowerCAmelCase	,  __lowerCAmelCase							)
				    or isinstance(__lowerCAmelCase	,  torch.IntTensor							)
				    or isinstance(__lowerCAmelCase	,  torch.LongTensor							)
				):
					raise ValueError(
					    (
					        '''Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to'''
					        F' `{self.__class__}.step()` is not supported. Make sure to pass'
					        ''' one of the `scheduler.timesteps` as a timestep.'''
					    )	,  )

				if not self.is_scale_input_called:
					logger.warning(
					    '''The `scale_model_input` function should be called before `step` to ensure correct denoising. '''
					    '''See `StableDiffusionPipeline` for a usage example.'''							)

				if isinstance(__lowerCAmelCase	,  torch.Tensor							):
					lowerCamelCase__        =		timestep.to(self.timesteps.device							)

				lowerCamelCase__        =		self.config.sigma_min
				lowerCamelCase__        =		self.config.sigma_max

				lowerCamelCase__        =		self.index_for_timestep(__lowerCAmelCase							)

				# sigma_next corresponds to next_t in original implementation
				lowerCamelCase__        =		self.sigmas[step_index]
				if step_index + 1 < self.config.num_train_timesteps:
					lowerCamelCase__        =		self.sigmas[step_index + 1]
				else:
					# Set sigma_next to sigma_min
					lowerCamelCase__        =		self.sigmas[-1]

				# Get scalings for boundary conditions
				lowerCamelCase__    ,     lowerCamelCase__        =		self.get_scalings_for_boundary_condition(__lowerCAmelCase							)

				# 1. Denoise model output using boundary conditions
				lowerCamelCase__        =		c_out * model_output + c_skip * sample
				if self.config.clip_denoised:
					lowerCamelCase__        =		denoised.clamp(-1	,  1							)

				# 2. Sample z ~ N(0, s_noise^2 * I)
				# Noise is not used for onestep sampling.
				if len(self.timesteps							) > 1:
					lowerCamelCase__        =		randn_tensor(
					    model_output.shape	,  dtype=model_output.dtype	,  device=model_output.device	,  generator=__lowerCAmelCase							)
				else:
					lowerCamelCase__        =		torch.zeros_like(__lowerCAmelCase							)
				lowerCamelCase__        =		noise * self.config.s_noise

				lowerCamelCase__        =		sigma_next.clamp(min=__lowerCAmelCase	,  max=__lowerCAmelCase							)

				# 3. Return noisy sample
				# tau = sigma_hat, eps = sigma_min
				lowerCamelCase__        =		denoised + z * (sigma_hat**2 - sigma_min**2) ** 0.5

				if not return_dict:
					return (prev_sample,)

				return CMStochasticIterativeSchedulerOutput(prev_sample=__lowerCAmelCase							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		self.sigmas.to(device=original_samples.device	,  dtype=original_samples.dtype							)
				if original_samples.device.type == "mps" and torch.is_floating_point(__lowerCAmelCase							):
					# mps does not support float64
					lowerCamelCase__        =		self.timesteps.to(original_samples.device	,  dtype=torch.floataa							)
					lowerCamelCase__        =		timesteps.to(original_samples.device	,  dtype=torch.floataa							)
				else:
					lowerCamelCase__        =		self.timesteps.to(original_samples.device							)
					lowerCamelCase__        =		timesteps.to(original_samples.device							)

				lowerCamelCase__        =		[(schedule_timesteps == t).nonzero().item() for t in timesteps]

				lowerCamelCase__        =		sigmas[step_indices].flatten()
				while len(sigma.shape							) < len(original_samples.shape							):
					lowerCamelCase__        =		sigma.unsqueeze(-1							)

				lowerCamelCase__        =		original_samples + noise * sigma
				return noisy_samples





			def __len__(   self							):



				'''simple docstring'''







				return self.config.num_train_timesteps



