





'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

snake_case_       :    Union[str, Any]          =					logging.get_logger(__name__)

snake_case_       :    Tuple          =					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

snake_case_       :    Dict          =					{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

snake_case_       :    Tuple          =					{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}





@lru_cache()
def     lowerCamelCase_ (					)	->		Dict:
						UpperCAmelCase_     : List[str]         =					(
						    list(range(ord('''!'''				),     ord('''~'''				) + 1				)				) + list(range(ord('''¡'''				),     ord('''¬'''				) + 1				)				) + list(range(ord('''®'''				),     ord('''ÿ'''				) + 1				)				)
						)
						UpperCAmelCase_     : List[str]         =					bs[:]
						UpperCAmelCase_     : Optional[Any]         =					0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(SCREAMING_SNAKE_CASE__				)
																		cs.append(2**8 + n				)
																		n += 1
						UpperCAmelCase_     : str         =					[chr(SCREAMING_SNAKE_CASE__				) for n in cs]
						return dict(zip(SCREAMING_SNAKE_CASE__,     SCREAMING_SNAKE_CASE__				)				)





def     lowerCamelCase_ (					SCREAMING_SNAKE_CASE__ :     Tuple				)	->		List[str]:
						UpperCAmelCase_     : int         =					set()
						UpperCAmelCase_     : Any         =					word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												UpperCAmelCase_     : Any         =					char
						return pairs


class    __a (lowerCamelCase       ):
							__a   :				Union[str, Any]       =    VOCAB_FILES_NAMES
							__a   :				Union[str, Any]       =    PRETRAINED_VOCAB_FILES_MAP
							__a   :				List[Any]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a   :				Dict       =    ["input_ids", "attention_mask"]

							def __init__(							self							:       Dict    ,       __magic_name__							:       Union[str, Any]    ,       __magic_name__							:       Union[str, Any]    ,       __magic_name__							:       Any="replace"    ,       __magic_name__							:       Optional[int]="<|endoftext|>"    ,       __magic_name__							:       Tuple="<|endoftext|>"    ,       __magic_name__							:       List[str]="<|endoftext|>"    ,       __magic_name__							:       List[Any]=None    ,       __magic_name__							:       Dict=False    ,       __magic_name__							:       List[Any]=False    ,       **__magic_name__							:       Union[str, Any]    ,       )   ->    Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_     : int         =					AddedToken(__magic_name__    ,       lstrip=__magic_name__    ,       rstrip=__magic_name__				) if isinstance(__magic_name__    ,       __magic_name__				) else bos_token
													UpperCAmelCase_     : Optional[Any]         =					AddedToken(__magic_name__    ,       lstrip=__magic_name__    ,       rstrip=__magic_name__				) if isinstance(__magic_name__    ,       __magic_name__				) else eos_token
													UpperCAmelCase_     : Tuple         =					AddedToken(__magic_name__    ,       lstrip=__magic_name__    ,       rstrip=__magic_name__				) if isinstance(__magic_name__    ,       __magic_name__				) else unk_token
													UpperCAmelCase_     : Optional[int]         =					AddedToken(__magic_name__    ,       lstrip=__magic_name__    ,       rstrip=__magic_name__				) if isinstance(__magic_name__    ,       __magic_name__				) else pad_token
													super().__init__(
													    errors=__magic_name__    ,       unk_token=__magic_name__    ,       bos_token=__magic_name__    ,       eos_token=__magic_name__    ,       pad_token=__magic_name__    ,       add_prefix_space=__magic_name__    ,       add_bos_token=__magic_name__    ,       **__magic_name__    ,       )
													UpperCAmelCase_     : List[Any]         =					add_bos_token

													with open(__magic_name__    ,       encoding='''utf-8'''				) as vocab_handle:
																			UpperCAmelCase_     : List[Any]         =					json.load(__magic_name__				)
													UpperCAmelCase_     : Optional[Any]         =					{v: k for k, v in self.encoder.items()}
													UpperCAmelCase_     : List[Any]         =					errors  # how to handle errors in decoding
													UpperCAmelCase_     : str         =					bytes_to_unicode()
													UpperCAmelCase_     : str         =					{v: k for k, v in self.byte_encoder.items()}
													with open(__magic_name__    ,       encoding='''utf-8'''				) as merges_handle:
																			UpperCAmelCase_     : List[Any]         =					merges_handle.read().split('''\n'''				)[1:-1]
													UpperCAmelCase_     : Any         =					[tuple(merge.split()				) for merge in bpe_merges]
													UpperCAmelCase_     : Dict         =					dict(zip(__magic_name__    ,       range(len(__magic_name__				)				)				)				)
													UpperCAmelCase_     : Any         =					{}
													UpperCAmelCase_     : Dict         =					add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													UpperCAmelCase_     : int         =					re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)

							@property
							def       UpperCAmelCase__		(							self							:       str				)   ->    str:







													"""simple docstring"""
													return len(self.encoder				)

							def       UpperCAmelCase__		(							self							:       str				)   ->    int:







													"""simple docstring"""
													return dict(self.encoder    ,       **self.added_tokens_encoder				)

							def       UpperCAmelCase__		(							self							:       Optional[int]    ,       __magic_name__							:       Tuple				)   ->    int:







													"""simple docstring"""
													if token in self.cache:
																			return self.cache[token]
													UpperCAmelCase_     : int         =					tuple(__magic_name__				)
													UpperCAmelCase_     : List[Any]         =					get_pairs(__magic_name__				)

													if not pairs:
																			return token

													while True:
																			UpperCAmelCase_     : Dict         =					min(__magic_name__    ,       key=lambda __magic_name__				: self.bpe_ranks.get(__magic_name__    ,       float('''inf'''				)				)				)
																			if bigram not in self.bpe_ranks:
																									break
																			UpperCAmelCase_  ,    UpperCAmelCase_     : Any         =					bigram
																			UpperCAmelCase_     : Dict         =					[]
																			UpperCAmelCase_     : Tuple         =					0
																			while i < len(__magic_name__				):
																									try:
																															UpperCAmelCase_     : int         =					word.index(__magic_name__    ,       __magic_name__				)
																									except ValueError:
																															new_word.extend(word[i:]				)
																															break
																									else:
																															new_word.extend(word[i:j]				)
																															UpperCAmelCase_     : int         =					j

																									if word[i] == first and i < len(__magic_name__				) - 1 and word[i + 1] == second:
																															new_word.append(first + second				)
																															i += 2
																									else:
																															new_word.append(word[i]				)
																															i += 1
																			UpperCAmelCase_     : Dict         =					tuple(__magic_name__				)
																			UpperCAmelCase_     : Dict         =					new_word
																			if len(__magic_name__				) == 1:
																									break
																			else:
																									UpperCAmelCase_     : int         =					get_pairs(__magic_name__				)
													UpperCAmelCase_     : Dict         =					''' '''.join(__magic_name__				)
													UpperCAmelCase_     : Union[str, Any]         =					word
													return word

							def       UpperCAmelCase__		(							self							:       Any    ,       __magic_name__							:       Tuple    ,       __magic_name__							:       Any=None				)   ->    Optional[Any]:







													"""simple docstring"""
													if self.add_bos_token:
																			UpperCAmelCase_     : List[Any]         =					[self.bos_token_id]
													else:
																			UpperCAmelCase_     : str         =					[]

													UpperCAmelCase_     : List[Any]         =					bos_token_ids + token_ids_a

													if token_ids_a is None:
																			return output

													return output + bos_token_ids + token_ids_a

							def       UpperCAmelCase__		(							self							:       List[str]    ,       __magic_name__							:       List[int]    ,       __magic_name__							:       Optional[List[int]] = None    ,       __magic_name__							:       bool = False				)   ->    List[int]:







													"""simple docstring"""
													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__magic_name__    ,       token_ids_a=__magic_name__    ,       already_has_special_tokens=__magic_name__				)

													if not self.add_bos_token:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__magic_name__    ,       token_ids_a=__magic_name__    ,       already_has_special_tokens=__magic_name__				)

													if token_ids_a is None:
																			return [1] + ([0] * len(__magic_name__				))
													return [1] + ([0] * len(__magic_name__				)) + [1] + ([0] * len(__magic_name__				))

							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       Tuple				)   ->    str:







													"""simple docstring"""
													UpperCAmelCase_     : str         =					[]
													for token in re.findall(self.pat    ,       __magic_name__				):
																			UpperCAmelCase_     : List[str]         =					''''''.join(
																			    self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__				).split(''' '''				)				)
													return bpe_tokens

							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       Union[str, Any]				)   ->    Any:







													"""simple docstring"""
													return self.encoder.get(__magic_name__    ,       self.encoder.get(self.unk_token				)				)

							def       UpperCAmelCase__		(							self							:       Tuple    ,       __magic_name__							:       Union[str, Any]				)   ->    Optional[int]:







													"""simple docstring"""
													return self.decoder.get(__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Dict    ,       __magic_name__							:       str				)   ->    Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_     : List[str]         =					''''''.join(__magic_name__				)
													UpperCAmelCase_     : Any         =					bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''    ,       errors=self.errors				)
													return text

							def       UpperCAmelCase__		(							self							:       List[str]    ,       __magic_name__							:       str    ,       __magic_name__							:       Optional[str] = None				)   ->    Tuple[str]:







													"""simple docstring"""
													if not os.path.isdir(__magic_name__				):
																			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
																			return
													UpperCAmelCase_     : int         =					os.path.join(
													    __magic_name__    ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
													UpperCAmelCase_     : Optional[int]         =					os.path.join(
													    __magic_name__    ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

													with open(__magic_name__    ,       '''w'''    ,       encoding='''utf-8'''				) as f:
																			f.write(json.dumps(self.encoder    ,       indent=2    ,       sort_keys=__magic_name__    ,       ensure_ascii=__magic_name__				) + '''\n'''				)

													UpperCAmelCase_     : Optional[Any]         =					0
													with open(__magic_name__    ,       '''w'''    ,       encoding='''utf-8'''				) as writer:
																			writer.write('''#version: 0.2\n'''				)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,       key=lambda __magic_name__				: kv[1]				):
																									if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																															    ''' Please check that the tokenizer is not corrupted!'''				)
																															UpperCAmelCase_     : Any         =					token_index
																									writer.write(''' '''.join(__magic_name__				) + '''\n'''				)
																									index += 1

													return vocab_file, merge_file

							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       str    ,       __magic_name__							:       Optional[int]=False    ,       **__magic_name__							:       int				)   ->    int:







													"""simple docstring"""
													UpperCAmelCase_     : int         =					kwargs.pop('''add_prefix_space'''    ,       self.add_prefix_space				)
													if is_split_into_words or add_prefix_space:
																			UpperCAmelCase_     : Dict         =					''' ''' + text
													return (text, kwargs)



							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       "Conversation"				)   ->    List[int]:







													"""simple docstring"""
													UpperCAmelCase_     : List[str]         =					[]
													for is_user, text in conversation.iter_texts():
																			input_ids.extend(self.encode(__magic_name__    ,       add_special_tokens=__magic_name__				) + [self.eos_token_id]				)
													if len(__magic_name__				) > self.model_max_length:
																			UpperCAmelCase_     : Optional[int]         =					input_ids[-self.model_max_length :]
													return input_ids












'''simple docstring'''







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
					import PIL


snake_case_       :    int          =					logging.get_logger(__name__)


class    __a (lowerCamelCase       ):
							__a   :				Optional[Any]       =    ["pixel_values"]

							def __init__(							self							:       str    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Dict[str, int] = None    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Dict[str, int] = None    ,       __magic_name__							:       PILImageResampling = PILImageResampling.BICUBIC    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Union[int, float] = 1 / 2_55    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       **__magic_name__							:       List[Any]    ,       )   ->    None:







													"""simple docstring"""
													super().__init__(**__magic_name__				)
													UpperCAmelCase_     : Tuple         =					crop_size if crop_size is not None else {'''height''': 2_56, '''width''': 2_56}
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)
													UpperCAmelCase_     : List[Any]         =					size if size is not None else {'''height''': 2_24, '''width''': 2_24}
													UpperCAmelCase_     : int         =					get_size_dict(__magic_name__				)

													UpperCAmelCase_     : Optional[int]         =					do_center_crop
													UpperCAmelCase_     : List[str]         =					crop_size
													UpperCAmelCase_     : Dict         =					do_resize
													UpperCAmelCase_     : str         =					size
													UpperCAmelCase_     : int         =					resample
													UpperCAmelCase_     : Optional[int]         =					do_rescale
													UpperCAmelCase_     : List[str]         =					rescale_factor
													UpperCAmelCase_     : int         =					do_normalize
													UpperCAmelCase_     : Any         =					image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
													UpperCAmelCase_     : str         =					image_std if image_std is not None else IMAGENET_DEFAULT_STD

							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Dict[str, int]    ,       __magic_name__							:       Optional[int] = None    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Union[str, Any]    ,       )   ->    np.ndarray:







													"""simple docstring"""
													UpperCAmelCase_     : List[Any]         =					self.size if size is None else size
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__				)
													UpperCAmelCase_     : Union[str, Any]         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)

													UpperCAmelCase_  ,    UpperCAmelCase_     : Any         =					get_image_size(__magic_name__				)
													UpperCAmelCase_     : List[str]         =					min(__magic_name__    ,       __magic_name__				)
													UpperCAmelCase_     : Dict         =					(size['''height'''] / crop_size['''height''']) * min_dim
													UpperCAmelCase_     : List[str]         =					(size['''width'''] / crop_size['''width''']) * min_dim
													return center_crop(__magic_name__    ,       size=(cropped_height, cropped_width)    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Union[str, Any]    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Dict[str, int]    ,       __magic_name__							:       PILImageResampling = PIL.Image.BICUBIC    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       str    ,       )   ->    np.ndarray:







													"""simple docstring"""
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__				)
													if "height" not in size or "width" not in size:
																			raise ValueError(F"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""				)
													return resize(
													    __magic_name__    ,       size=(size['''height'''], size['''width'''])    ,       resample=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Tuple    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Union[int, float]    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Tuple    ,       )   ->    str:







													"""simple docstring"""
													return rescale(__magic_name__    ,       scale=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Optional[int]    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Union[float, List[float]]    ,       __magic_name__							:       Union[float, List[float]]    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Dict    ,       )   ->    np.ndarray:







													"""simple docstring"""
													return normalize(__magic_name__    ,       mean=__magic_name__    ,       std=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)



							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       ImageInput    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Dict[str, int]] = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Dict[str, int]] = None    ,       __magic_name__							:       PILImageResampling = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[float] = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[str, TensorType]] = None    ,       __magic_name__							:       ChannelDimension = ChannelDimension.FIRST    ,       **__magic_name__							:       Optional[Any]    ,       )   ->    PIL.Image.Image:







													"""simple docstring"""
													UpperCAmelCase_     : List[str]         =					do_center_crop if do_center_crop is not None else self.do_center_crop
													UpperCAmelCase_     : str         =					crop_size if crop_size is not None else self.crop_size
													UpperCAmelCase_     : int         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)
													UpperCAmelCase_     : Dict         =					do_resize if do_resize is not None else self.do_resize
													UpperCAmelCase_     : Optional[Any]         =					size if size is not None else self.size
													UpperCAmelCase_     : Tuple         =					get_size_dict(__magic_name__				)
													UpperCAmelCase_     : Any         =					resample if resample is not None else self.resample
													UpperCAmelCase_     : int         =					do_rescale if do_rescale is not None else self.do_rescale
													UpperCAmelCase_     : str         =					rescale_factor if rescale_factor is not None else self.rescale_factor
													UpperCAmelCase_     : Optional[Any]         =					do_normalize if do_normalize is not None else self.do_normalize
													UpperCAmelCase_     : Tuple         =					image_mean if image_mean is not None else self.image_mean
													UpperCAmelCase_     : Tuple         =					image_std if image_std is not None else self.image_std

													UpperCAmelCase_     : Optional[Any]         =					make_list_of_images(__magic_name__				)

													if not valid_images(__magic_name__				):
																			raise ValueError(
																			    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																			    '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

													if do_center_crop and crop_size is None:
																			raise ValueError('''If `do_center_crop` is set to `True`, `crop_size` must be provided.'''				)

													if do_resize and size is None:
																			raise ValueError('''Size must be specified if do_resize is True.'''				)

													if do_rescale and rescale_factor is None:
																			raise ValueError('''Rescale factor must be specified if do_rescale is True.'''				)

													if do_normalize and (image_mean is None or image_std is None):
																			raise ValueError('''Image mean and image standard deviation must be specified if do_normalize is True.'''				)

													# All transformations expect numpy arrays.
													UpperCAmelCase_     : Tuple         =					[to_numpy_array(__magic_name__				) for image in images]

													if do_center_crop:
																			UpperCAmelCase_     : int         =					[self.center_crop(__magic_name__    ,       __magic_name__    ,       size=__magic_name__				) for image in images]

													if do_resize:
																			UpperCAmelCase_     : Dict         =					[self.resize(image=__magic_name__    ,       size=__magic_name__    ,       resample=__magic_name__				) for image in images]

													if do_rescale:
																			UpperCAmelCase_     : Union[str, Any]         =					[self.rescale(image=__magic_name__    ,       scale=__magic_name__				) for image in images]

													if do_normalize:
																			UpperCAmelCase_     : Any         =					[self.normalize(image=__magic_name__    ,       mean=__magic_name__    ,       std=__magic_name__				) for image in images]

													UpperCAmelCase_     : Optional[int]         =					[to_channel_dimension_format(__magic_name__    ,       __magic_name__				) for image in images]

													UpperCAmelCase_     : Union[str, Any]         =					{'''pixel_values''': images}
													return BatchFeature(data=__magic_name__    ,       tensor_type=__magic_name__				)





