





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation

_a									=				logging.get_logger(__name__)

_a									=				{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_a									=				{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

_a									=				{
    "gpt2": 1_024,
    "gpt2-medium": 1_024,
    "gpt2-large": 1_024,
    "gpt2-xl": 1_024,
    "distilgpt2": 1_024,
}





@lru_cache()
def lowerCAmelCase__()    ->				Any:







	'''simple docstring'''







	lowerCamelCase__        =		(
	    list(range(ord('''!'''							)	,ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)	,ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)	,ord('''ÿ'''							) + 1							)							)
	)
	lowerCamelCase__        =		bs[:]
	lowerCamelCase__        =		0
	for b in range(2**8							):
		if b not in bs:
			bs.append(__snake_case							)
			cs.append(2**8 + n							)
			n += 1
	lowerCamelCase__        =		[chr(__snake_case							) for n in cs]
	return dict(zip(__snake_case	,__snake_case							)							)





def lowerCAmelCase__(__snake_case							)    ->				Tuple:







	'''simple docstring'''







	lowerCamelCase__        =		set()
	lowerCamelCase__        =		word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)							)
		lowerCamelCase__        =		char
	return pairs






class  __A   (   lowerCAmelCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						VOCAB_FILES_NAMES
			lowerCAmelCase_												=						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_												=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_												=						["""input_ids""", """attention_mask"""]





			def __init__(   self	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase="replace"	,  __lowerCAmelCase="<|endoftext|>"	,  __lowerCAmelCase="<|endoftext|>"	,  __lowerCAmelCase="<|endoftext|>"	,  __lowerCAmelCase=None	,  __lowerCAmelCase=False	,  __lowerCAmelCase=False	,  **__lowerCAmelCase	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else bos_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else eos_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else unk_token
				lowerCamelCase__        =		AddedToken(__lowerCAmelCase	,  lstrip=__lowerCAmelCase	,  rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase	,  __lowerCAmelCase							) else pad_token
				super().__init__(
				    errors=__lowerCAmelCase	,  unk_token=__lowerCAmelCase	,  bos_token=__lowerCAmelCase	,  eos_token=__lowerCAmelCase	,  pad_token=__lowerCAmelCase	,  add_prefix_space=__lowerCAmelCase	,  add_bos_token=__lowerCAmelCase	,  **__lowerCAmelCase	,  )
				lowerCamelCase__        =		add_bos_token

				with open(__lowerCAmelCase	,  encoding='''utf-8'''							) as vocab_handle:
					lowerCamelCase__        =		json.load(__lowerCAmelCase							)
				lowerCamelCase__        =		{v: k for k, v in self.encoder.items()}
				lowerCamelCase__        =		errors  # how to handle errors in decoding
				lowerCamelCase__        =		bytes_to_unicode()
				lowerCamelCase__        =		{v: k for k, v in self.byte_encoder.items()}
				with open(__lowerCAmelCase	,  encoding='''utf-8'''							) as merges_handle:
					lowerCamelCase__        =		merges_handle.read().split('''\n'''							)[1:-1]
				lowerCamelCase__        =		[tuple(merge.split()							) for merge in bpe_merges]
				lowerCamelCase__        =		dict(zip(__lowerCAmelCase	,  range(len(__lowerCAmelCase							)							)							)							)
				lowerCamelCase__        =		{}
				lowerCamelCase__        =		add_prefix_space

				# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
				lowerCamelCase__        =		re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''							)





			@property
			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return len(self.encoder							)





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return dict(self.encoder	,  **self.added_tokens_encoder							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				if token in self.cache:
					return self.cache[token]
				lowerCamelCase__        =		tuple(__lowerCAmelCase							)
				lowerCamelCase__        =		get_pairs(__lowerCAmelCase							)

				if not pairs:
					return token

				while True:
					lowerCamelCase__        =		min(__lowerCAmelCase	,  key=lambda __lowerCAmelCase							: self.bpe_ranks.get(__lowerCAmelCase	,  float('''inf'''							)							)							)
					if bigram not in self.bpe_ranks:
						break
					lowerCamelCase__    ,     lowerCamelCase__        =		bigram
					lowerCamelCase__        =		[]
					lowerCamelCase__        =		0
					while i < len(__lowerCAmelCase							):
						try:
							lowerCamelCase__        =		word.index(__lowerCAmelCase	,  __lowerCAmelCase							)
						except ValueError:
							new_word.extend(word[i:]							)
							break
						else:
							new_word.extend(word[i:j]							)
							lowerCamelCase__        =		j

						if word[i] == first and i < len(__lowerCAmelCase							) - 1 and word[i + 1] == second:
							new_word.append(first + second							)
							i += 2
						else:
							new_word.append(word[i]							)
							i += 1
					lowerCamelCase__        =		tuple(__lowerCAmelCase							)
					lowerCamelCase__        =		new_word
					if len(__lowerCAmelCase							) == 1:
						break
					else:
						lowerCamelCase__        =		get_pairs(__lowerCAmelCase							)
				lowerCamelCase__        =		''' '''.join(__lowerCAmelCase							)
				lowerCamelCase__        =		word
				return word





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase=None							):



				'''simple docstring'''







				if self.add_bos_token:
					lowerCamelCase__        =		[self.bos_token_id]
				else:
					lowerCamelCase__        =		[]

				lowerCamelCase__        =		bos_token_ids + token_ids_a

				if token_ids_a is None:
					return output

				return output + bos_token_ids + token_ids_a





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase = None	,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__lowerCAmelCase	,  token_ids_a=__lowerCAmelCase	,  already_has_special_tokens=__lowerCAmelCase							)

				if not self.add_bos_token:
					return super().get_special_tokens_mask(
					    token_ids_a=__lowerCAmelCase	,  token_ids_a=__lowerCAmelCase	,  already_has_special_tokens=__lowerCAmelCase							)

				if token_ids_a is None:
					return [1] + ([0] * len(__lowerCAmelCase							))
				return [1] + ([0] * len(__lowerCAmelCase							)) + [1] + ([0] * len(__lowerCAmelCase							))





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		[]
				for token in re.findall(self.pat	,  __lowerCAmelCase							):
					lowerCamelCase__        =		''''''.join(
					    self.byte_encoder[b] for b in token.encode('''utf-8'''							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase							).split(''' '''							)							)
				return bpe_tokens





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.encoder.get(__lowerCAmelCase	,  self.encoder.get(self.unk_token							)							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.decoder.get(__lowerCAmelCase							)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		''''''.join(__lowerCAmelCase							)
				lowerCamelCase__        =		bytearray([self.byte_decoder[c] for c in text]							).decode('''utf-8'''	,  errors=self.errors							)
				return text





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if not os.path.isdir(__lowerCAmelCase							):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
					return
				lowerCamelCase__        =		os.path.join(
				    __lowerCAmelCase	,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
				lowerCamelCase__        =		os.path.join(
				    __lowerCAmelCase	,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']							)

				with open(__lowerCAmelCase	,  '''w'''	,  encoding='''utf-8'''							) as f:
					f.write(json.dumps(self.encoder	,  indent=2	,  sort_keys=__lowerCAmelCase	,  ensure_ascii=__lowerCAmelCase							) + '''\n'''							)

				lowerCamelCase__        =		0
				with open(__lowerCAmelCase	,  '''w'''	,  encoding='''utf-8'''							) as writer:
					writer.write('''#version: 0.2\n'''							)
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,  key=lambda __lowerCAmelCase							: kv[1]							):
						if index != token_index:
							logger.warning(
							    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
							    ''' Please check that the tokenizer is not corrupted!'''							)
							lowerCamelCase__        =		token_index
						writer.write(''' '''.join(__lowerCAmelCase							) + '''\n'''							)
						index += 1

				return vocab_file, merge_file





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase=False	,  **__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		kwargs.pop('''add_prefix_space'''	,  self.add_prefix_space							)
				if is_split_into_words or add_prefix_space:
					lowerCamelCase__        =		''' ''' + text
				return (text, kwargs)





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__        =		[]
				for is_user, text in conversation.iter_texts():
					input_ids.extend(self.encode(__lowerCAmelCase	,  add_special_tokens=__lowerCAmelCase							) + [self.eos_token_id]							)
				if len(__lowerCAmelCase							) > self.model_max_length:
					lowerCamelCase__        =		input_ids[-self.model_max_length :]
				return input_ids










import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import ViltImageProcessor






class  __A   (   unittest.TestCase			):

			'''simple docstring'''







			def __init__(   self	,  __lowerCAmelCase	,  __lowerCAmelCase=7	,  __lowerCAmelCase=3	,  __lowerCAmelCase=1_8	,  __lowerCAmelCase=3_0	,  __lowerCAmelCase=4_0_0	,  __lowerCAmelCase=True	,  __lowerCAmelCase=None	,  __lowerCAmelCase=2	,  __lowerCAmelCase=True	,  __lowerCAmelCase=[0.5, 0.5, 0.5]	,  __lowerCAmelCase=[0.5, 0.5, 0.5]	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		size if size is not None else {'''shortest_edge''': 3_0}
				lowerCamelCase__        =		parent
				lowerCamelCase__        =		batch_size
				lowerCamelCase__        =		num_channels
				lowerCamelCase__        =		image_size
				lowerCamelCase__        =		min_resolution
				lowerCamelCase__        =		max_resolution
				lowerCamelCase__        =		do_resize
				lowerCamelCase__        =		size
				lowerCamelCase__        =		size_divisor
				lowerCamelCase__        =		do_normalize
				lowerCamelCase__        =		image_mean
				lowerCamelCase__        =		image_std





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return {
				    "image_mean": self.image_mean,
				    "image_std": self.image_std,
				    "do_normalize": self.do_normalize,
				    "do_resize": self.do_resize,
				    "size": self.size,
				    "size_divisor": self.size_divisor,
				}





			def 		__lowerCamelCase    (   self	,  __lowerCAmelCase	,  __lowerCAmelCase=False							):



				'''simple docstring'''







				if not batched:
					lowerCamelCase__        =		self.size['''shortest_edge''']
					lowerCamelCase__        =		image_inputs[0]
					if isinstance(__lowerCAmelCase	,  Image.Image							):
						lowerCamelCase__    ,     lowerCamelCase__        =		image.size
					else:
						lowerCamelCase__    ,     lowerCamelCase__        =		image.shape[1], image.shape[2]
					lowerCamelCase__        =		size / min(__lowerCAmelCase	,  __lowerCAmelCase							)
					if h < w:
						lowerCamelCase__    ,     lowerCamelCase__        =		size, scale * w
					else:
						lowerCamelCase__    ,     lowerCamelCase__        =		scale * h, size

					lowerCamelCase__        =		int((1_3_3_3 / 8_0_0) * size							)
					if max(__lowerCAmelCase	,  __lowerCAmelCase							) > max_size:
						lowerCamelCase__        =		max_size / max(__lowerCAmelCase	,  __lowerCAmelCase							)
						lowerCamelCase__        =		newh * scale
						lowerCamelCase__        =		neww * scale

					lowerCamelCase__    ,     lowerCamelCase__        =		int(newh + 0.5							), int(neww + 0.5							)
					lowerCamelCase__    ,     lowerCamelCase__        =		(
					    newh // self.size_divisor * self.size_divisor,
					    neww // self.size_divisor * self.size_divisor,
					)

				else:
					lowerCamelCase__        =		[]
					for image in image_inputs:
						lowerCamelCase__    ,     lowerCamelCase__        =		self.get_expected_values([image]							)
						expected_values.append((expected_height, expected_width)							)
					lowerCamelCase__        =		max(__lowerCAmelCase	,  key=lambda __lowerCAmelCase							: item[0]							)[0]
					lowerCamelCase__        =		max(__lowerCAmelCase	,  key=lambda __lowerCAmelCase							: item[1]							)[1]

				return expected_height, expected_width






@require_torch
@require_vision
class  __A   (   lowerCAmelCase      ,    unittest.TestCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						ViltImageProcessor if is_vision_available() else None





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		ViltImageProcessingTester(self							)





			@property
			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return self.image_processor_tester.prepare_image_processor_dict()





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.image_processing_class(**self.image_processor_dict							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''image_mean'''							)							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''image_std'''							)							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''do_normalize'''							)							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''do_resize'''							)							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''size'''							)							)
				self.assertTrue(hasattr(__lowerCAmelCase	,  '''size_divisor'''							)							)





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.image_processing_class.from_dict(self.image_processor_dict							)
				self.assertEqual(image_processor.size	,  {'''shortest_edge''': 3_0}							)

				lowerCamelCase__        =		self.image_processing_class.from_dict(self.image_processor_dict	,  size=4_2							)
				self.assertEqual(image_processor.size	,  {'''shortest_edge''': 4_2}							)





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				pass





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.image_processing_class(**self.image_processor_dict							)
				# create random PIL images
				lowerCamelCase__        =		prepare_image_inputs(self.image_processor_tester	,  equal_resolution=__lowerCAmelCase							)
				for image in image_inputs:
					self.assertIsInstance(__lowerCAmelCase	,  Image.Image							)

				# Test not batched input
				lowerCamelCase__        =		image_processing(image_inputs[0]	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)	,  )

				# Test batched
				lowerCamelCase__        =		image_processing(__lowerCAmelCase	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase	,  batched=__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        expected_height,
				        expected_width,
				    )	,  )





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.image_processing_class(**self.image_processor_dict							)
				# create random numpy tensors
				lowerCamelCase__        =		prepare_image_inputs(self.image_processor_tester	,  equal_resolution=__lowerCAmelCase	,  numpify=__lowerCAmelCase							)
				for image in image_inputs:
					self.assertIsInstance(__lowerCAmelCase	,  np.ndarray							)

				# Test not batched input
				lowerCamelCase__        =		image_processing(image_inputs[0]	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)	,  )

				# Test batched
				lowerCamelCase__        =		image_processing(__lowerCAmelCase	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase	,  batched=__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        expected_height,
				        expected_width,
				    )	,  )





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				lowerCamelCase__        =		self.image_processing_class(**self.image_processor_dict							)
				# create random PyTorch tensors
				lowerCamelCase__        =		prepare_image_inputs(self.image_processor_tester	,  equal_resolution=__lowerCAmelCase	,  torchify=__lowerCAmelCase							)
				for image in image_inputs:
					self.assertIsInstance(__lowerCAmelCase	,  torch.Tensor							)

				# Test not batched input
				lowerCamelCase__        =		image_processing(image_inputs[0]	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (1, self.image_processor_tester.num_channels, expected_height, expected_width)	,  )

				# Test batched
				lowerCamelCase__        =		image_processing(__lowerCAmelCase	,  return_tensors='''pt'''							).pixel_values

				lowerCamelCase__    ,     lowerCamelCase__        =		self.image_processor_tester.get_expected_values(__lowerCAmelCase	,  batched=__lowerCAmelCase							)
				self.assertEqual(
				    encoded_images.shape	,  (
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        expected_height,
				        expected_width,
				    )	,  )



