



"""simple docstring"""
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

__a     =    logging.get_logger(__name__)

__a     =    {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__a     =    {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

__a     =    {
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}
@lru_cache()
def        A_  (      ):







						'''simple docstring'''


						snake_case_			:str						=      (
						    list(range(ord("""!"""     ), ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     ), ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     ), ord("""ÿ"""     ) + 1     )     )
						)
						snake_case_			:Union[str, Any]						=      bs[:]
						snake_case_			:Dict						=      0
						for b in range(2**8     ):
												if b not in bs:
																		bs.append(_lowercase     )
																		cs.append(2**8 + n     )
																		n += 1
						snake_case_			:List[str]						=      [chr(_lowercase     ) for n in cs]
						return dict(zip(_lowercase, _lowercase     )     )
def        A_  (      _lowercase     ):







						'''simple docstring'''


						snake_case_			:Union[str, Any]						=      set()
						snake_case_			:Optional[Any]						=      word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)     )
												snake_case_			:str						=      char
						return pairs

class     lowerCamelCase  (      _lowerCAmelCase    ):






					'''simple docstring'''





					_A		:      Optional[Any]						=					VOCAB_FILES_NAMES
					_A		:      List[str]						=					PRETRAINED_VOCAB_FILES_MAP
					_A		:      List[Any]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_A		:      int						=					["""input_ids""", """attention_mask"""]






					def __init__(	self:							Union[str, Any]						,					snake_case:							List[Any]						,					snake_case:							Optional[Any]						,					snake_case:							int="replace"						,					snake_case:							Optional[int]="<|endoftext|>"						,					snake_case:							Union[str, Any]="<|endoftext|>"						,					snake_case:							str="<|endoftext|>"						,					snake_case:							Dict=None						,					snake_case:							Dict=False						,					snake_case:							Union[str, Any]=False						,					**snake_case:							Tuple						,					)       ->     Tuple:
											snake_case_			:int						=      AddedToken(snake_case						,					lstrip=snake_case						,					rstrip=snake_case						) if isinstance(snake_case						,					snake_case						) else bos_token
											snake_case_			:int						=      AddedToken(snake_case						,					lstrip=snake_case						,					rstrip=snake_case						) if isinstance(snake_case						,					snake_case						) else eos_token
											snake_case_			:int						=      AddedToken(snake_case						,					lstrip=snake_case						,					rstrip=snake_case						) if isinstance(snake_case						,					snake_case						) else unk_token
											snake_case_			:int						=      AddedToken(snake_case						,					lstrip=snake_case						,					rstrip=snake_case						) if isinstance(snake_case						,					snake_case						) else pad_token
											super().__init__(
											    errors=snake_case						,					unk_token=snake_case						,					bos_token=snake_case						,					eos_token=snake_case						,					pad_token=snake_case						,					add_prefix_space=snake_case						,					add_bos_token=snake_case						,					**snake_case						,					)
											snake_case_			:Any						=      add_bos_token

											with open(snake_case						,					encoding="""utf-8"""						) as vocab_handle:
																	snake_case_			:str						=      json.load(snake_case						)
											snake_case_			:Tuple						=      {v: k for k, v in self.encoder.items()}
											snake_case_			:List[str]						=      errors  # how to handle errors in decoding
											snake_case_			:List[Any]						=      bytes_to_unicode()
											snake_case_			:Optional[int]						=      {v: k for k, v in self.byte_encoder.items()}
											with open(snake_case						,					encoding="""utf-8"""						) as merges_handle:
																	snake_case_			:Optional[int]						=      merges_handle.read().split("""\n"""						)[1:-1]
											snake_case_			:str						=      [tuple(merge.split()						) for merge in bpe_merges]
											snake_case_			:str						=      dict(zip(snake_case						,					range(len(snake_case						)						)						)						)
											snake_case_			:Any						=      {}
											snake_case_			:Tuple						=      add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											snake_case_			:int						=      re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)






					@property
					def        lowerCAmelCase_    (	self:							Dict						)       ->     Dict:
											return len(self.encoder						)






					def        lowerCAmelCase_    (	self:							Any						)       ->     Union[str, Any]:
											return dict(self.encoder						,					**self.added_tokens_encoder						)






					def        lowerCAmelCase_    (	self:							List[Any]						,					snake_case:							Union[str, Any]						)       ->     Union[str, Any]:
											if token in self.cache:
																	return self.cache[token]
											snake_case_			:Dict						=      tuple(snake_case						)
											snake_case_			:List[str]						=      get_pairs(snake_case						)

											if not pairs:
																	return token

											while True:
																	snake_case_			:Optional[int]						=      min(snake_case						,					key=lambda snake_case						: self.bpe_ranks.get(snake_case						,					float("""inf"""						)						)						)
																	if bigram not in self.bpe_ranks:
																							break
																	snake_case_,	snake_case_			:Optional[int]						=      bigram
																	snake_case_			:str						=      []
																	snake_case_			:Any						=      0
																	while i < len(snake_case						):
																							try:
																													snake_case_			:List[str]						=      word.index(snake_case						,					snake_case						)
																							except ValueError:
																													new_word.extend(word[i:]						)
																													break
																							else:
																													new_word.extend(word[i:j]						)
																													snake_case_			:str						=      j

																							if word[i] == first and i < len(snake_case						) - 1 and word[i + 1] == second:
																													new_word.append(first + second						)
																													i += 2
																							else:
																													new_word.append(word[i]						)
																													i += 1
																	snake_case_			:List[str]						=      tuple(snake_case						)
																	snake_case_			:Optional[Any]						=      new_word
																	if len(snake_case						) == 1:
																							break
																	else:
																							snake_case_			:Dict						=      get_pairs(snake_case						)
											snake_case_			:str						=      """ """.join(snake_case						)
											snake_case_			:Dict						=      word
											return word






					def        lowerCAmelCase_    (	self:							Any						,					snake_case:							Dict						,					snake_case:							Optional[int]=None						)       ->     Optional[Any]:
											if self.add_bos_token:
																	snake_case_			:Tuple						=      [self.bos_token_id]
											else:
																	snake_case_			:str						=      []

											snake_case_			:Optional[int]						=      bos_token_ids + token_ids_a

											if token_ids_a is None:
																	return output

											return output + bos_token_ids + token_ids_a






					def        lowerCAmelCase_    (	self:							Any						,					snake_case:							List[int]						,					snake_case:							Optional[List[int]] = None						,					snake_case:							bool = False						)       ->     List[int]:
											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case						,					token_ids_a=snake_case						,					already_has_special_tokens=snake_case						)

											if not self.add_bos_token:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case						,					token_ids_a=snake_case						,					already_has_special_tokens=snake_case						)

											if token_ids_a is None:
																	return [1] + ([0] * len(snake_case						))
											return [1] + ([0] * len(snake_case						)) + [1] + ([0] * len(snake_case						))






					def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							List[str]						)       ->     int:
											snake_case_			:Tuple						=      []
											for token in re.findall(self.pat						,					snake_case						):
																	snake_case_			:Tuple						=      """""".join(
																	    self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case						).split(""" """						)						)
											return bpe_tokens






					def        lowerCAmelCase_    (	self:							Union[str, Any]						,					snake_case:							List[Any]						)       ->     Tuple:
											return self.encoder.get(snake_case						,					self.encoder.get(self.unk_token						)						)






					def        lowerCAmelCase_    (	self:							Tuple						,					snake_case:							int						)       ->     str:
											return self.decoder.get(snake_case						)






					def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							Optional[Any]						)       ->     Tuple:
											snake_case_			:Any						=      """""".join(snake_case						)
											snake_case_			:Any						=      bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""						,					errors=self.errors						)
											return text






					def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							str						,					snake_case:							Optional[str] = None						)       ->     Tuple[str]:
											if not os.path.isdir(snake_case						):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																	return
											snake_case_			:str						=      os.path.join(
											    snake_case						,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
											snake_case_			:Optional[int]						=      os.path.join(
											    snake_case						,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

											with open(snake_case						,					"""w"""						,					encoding="""utf-8"""						) as f:
																	f.write(json.dumps(self.encoder						,					indent=2						,					sort_keys=snake_case						,					ensure_ascii=snake_case						) + """\n"""						)

											snake_case_			:Dict						=      0
											with open(snake_case						,					"""w"""						,					encoding="""utf-8"""						) as writer:
																	writer.write("""#version: 0.2\n"""						)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,					key=lambda snake_case						: kv[1]						):
																							if index != token_index:
																													logger.warning(
																													    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																													    """ Please check that the tokenizer is not corrupted!"""						)
																													snake_case_			:Optional[Any]						=      token_index
																							writer.write(""" """.join(snake_case						) + """\n"""						)
																							index += 1

											return vocab_file, merge_file






					def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							List[Any]						,					snake_case:							Optional[int]=False						,					**snake_case:							Optional[int]						)       ->     Optional[Any]:
											snake_case_			:str						=      kwargs.pop("""add_prefix_space"""						,					self.add_prefix_space						)
											if is_split_into_words or add_prefix_space:
																	snake_case_			:Dict						=      """ """ + text
											return (text, kwargs)






					def        lowerCAmelCase_    (	self:							int						,					snake_case:							"Conversation"						)       ->     List[int]:
											snake_case_			:Union[str, Any]						=      []
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(snake_case						,					add_special_tokens=snake_case						) + [self.eos_token_id]						)
											if len(snake_case						) > self.model_max_length:
																	snake_case_			:Dict						=      input_ids[-self.model_max_length :]
											return input_ids





"""simple docstring"""
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import ViTImageProcessor
class     lowerCamelCase  (      unittest.TestCase    ):






					'''simple docstring'''





					def __init__(	self:							int						,					snake_case:							List[str]						,					snake_case:							List[str]=7						,					snake_case:							Union[str, Any]=3						,					snake_case:							List[str]=18						,					snake_case:							Any=30						,					snake_case:							Any=400						,					snake_case:							Any=True						,					snake_case:							List[str]=None						,					snake_case:							Union[str, Any]=True						,					snake_case:							int=[0.5, 0.5, 0.5]						,					snake_case:							Tuple=[0.5, 0.5, 0.5]						,					)       ->     int:
											snake_case_			:Dict						=      size if size is not None else {"""height""": 18, """width""": 18}
											snake_case_			:Optional[Any]						=      parent
											snake_case_			:Tuple						=      batch_size
											snake_case_			:Tuple						=      num_channels
											snake_case_			:Optional[int]						=      image_size
											snake_case_			:Tuple						=      min_resolution
											snake_case_			:str						=      max_resolution
											snake_case_			:Any						=      do_resize
											snake_case_			:Union[str, Any]						=      size
											snake_case_			:Tuple						=      do_normalize
											snake_case_			:int						=      image_mean
											snake_case_			:List[Any]						=      image_std






					def        lowerCAmelCase_    (	self:							List[Any]						)       ->     List[Any]:
											return {
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											    "do_normalize": self.do_normalize,
											    "do_resize": self.do_resize,
											    "size": self.size,
											}

@require_torch
@require_vision
class     lowerCamelCase  (      _lowerCAmelCase   ,						unittest.TestCase    ):






					'''simple docstring'''





					_A		:      Tuple						=					ViTImageProcessor if is_vision_available() else None






					def        lowerCAmelCase_    (	self:							str						)       ->     Optional[Any]:
											snake_case_			:Dict						=      ViTImageProcessingTester(self						)






					@property
					def        lowerCAmelCase_    (	self:							Tuple						)       ->     Optional[Any]:
											return self.image_processor_tester.prepare_image_processor_dict()






					def        lowerCAmelCase_    (	self:							str						)       ->     List[str]:
											snake_case_			:List[str]						=      self.image_processing_class(**self.image_processor_dict						)
											self.assertTrue(hasattr(snake_case						,					"""image_mean"""						)						)
											self.assertTrue(hasattr(snake_case						,					"""image_std"""						)						)
											self.assertTrue(hasattr(snake_case						,					"""do_normalize"""						)						)
											self.assertTrue(hasattr(snake_case						,					"""do_resize"""						)						)
											self.assertTrue(hasattr(snake_case						,					"""size"""						)						)






					def        lowerCAmelCase_    (	self:							Dict						)       ->     Union[str, Any]:
											snake_case_			:int						=      self.image_processing_class.from_dict(self.image_processor_dict						)
											self.assertEqual(image_processor.size						,					{"""height""": 18, """width""": 18}						)

											snake_case_			:int						=      self.image_processing_class.from_dict(self.image_processor_dict						,					size=42						)
											self.assertEqual(image_processor.size						,					{"""height""": 42, """width""": 42}						)






					def        lowerCAmelCase_    (	self:							Tuple						)       ->     int:
											pass






					def        lowerCAmelCase_    (	self:							Dict						)       ->     Union[str, Any]:
											# Initialize image_processing
											snake_case_			:Any						=      self.image_processing_class(**self.image_processor_dict						)
											# create random PIL images
											snake_case_			:str						=      prepare_image_inputs(self.image_processor_tester						,					equal_resolution=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,					Image.Image						)

											# Test not batched input
											snake_case_			:List[str]						=      image_processing(image_inputs[0]						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)

											# Test batched
											snake_case_			:Dict						=      image_processing(snake_case						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)






					def        lowerCAmelCase_    (	self:							Optional[Any]						)       ->     Tuple:
											# Initialize image_processing
											snake_case_			:int						=      self.image_processing_class(**self.image_processor_dict						)
											# create random numpy tensors
											snake_case_			:str						=      prepare_image_inputs(self.image_processor_tester						,					equal_resolution=snake_case						,					numpify=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,					np.ndarray						)

											# Test not batched input
											snake_case_			:Any						=      image_processing(image_inputs[0]						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)

											# Test batched
											snake_case_			:Any						=      image_processing(snake_case						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)






					def        lowerCAmelCase_    (	self:							List[Any]						)       ->     Optional[int]:
											# Initialize image_processing
											snake_case_			:Union[str, Any]						=      self.image_processing_class(**self.image_processor_dict						)
											# create random PyTorch tensors
											snake_case_			:List[Any]						=      prepare_image_inputs(self.image_processor_tester						,					equal_resolution=snake_case						,					torchify=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,					torch.Tensor						)

											# Test not batched input
											snake_case_			:Optional[Any]						=      image_processing(image_inputs[0]						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)

											# Test batched
											snake_case_			:List[Any]						=      image_processing(snake_case						,					return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size["""height"""],
											        self.image_processor_tester.size["""width"""],
											    )						,					)
