





'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class    __a (__UpperCAmelCase    ,						unittest.TestCase       ):
							__a   :				Optional[Any]       =    GPTNeoXJapaneseTokenizer
							__a   :				Tuple       =    False
							__a   :				str       =    {"""do_clean_text""": False, """add_prefix_space""": False}

							def       UpperCAmelCase__		(							self							:       Any				)   ->    List[str]:







													"""simple docstring"""
													super().setUp()

													UpperCAmelCase_     : List[Any]         =					[
													    '''こん''',
													    '''こんに''',
													    '''にちは''',
													    '''ばんは''',
													    '''世界,㔺界''',
													    '''、''',
													    '''。''',
													    '''<BR>''',
													    '''<SP>''',
													    '''<TAB>''',
													    '''<URL>''',
													    '''<EMAIL>''',
													    '''<TEL>''',
													    '''<DATE>''',
													    '''<PRICE>''',
													    '''<BLOCK>''',
													    '''<KIGOU>''',
													    '''<U2000U2BFF>''',
													    '''<|emoji1|>''',
													    '''<unk>''',
													    '''<|startoftext|>''',
													    '''<|endoftext|>''',
													]
													UpperCAmelCase_     : Dict         =					{'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
													UpperCAmelCase_     : Any         =					{'''unk_token''': '''<unk>'''}

													UpperCAmelCase_     : List[str]         =					os.path.join(self.tmpdirname    ,       VOCAB_FILES_NAMES['''vocab_file''']				)
													UpperCAmelCase_     : List[Any]         =					os.path.join(self.tmpdirname    ,       VOCAB_FILES_NAMES['''emoji_file''']				)
													with open(self.vocab_file    ,       '''w'''    ,       encoding='''utf-8'''				) as vocab_writer:
																			vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)
													with open(self.emoji_file    ,       '''w'''				) as emoji_writer:
																			emoji_writer.write(json.dumps(lowerCamelCase__				)				)

							def       UpperCAmelCase__		(							self							:       Tuple    ,       **__magic_name__							:       Any				)   ->    Optional[Any]:







													"""simple docstring"""
													kwargs.update(self.special_tokens_map				)
													return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname    ,       **lowerCamelCase__				)

							def       UpperCAmelCase__		(							self							:       List[Any]    ,       __magic_name__							:       List[str]				)   ->    Any:







													"""simple docstring"""
													UpperCAmelCase_     : List[Any]         =					'''こんにちは、世界。 \nこんばんは、㔺界。😀'''
													UpperCAmelCase_     : Any         =					'''こんにちは、世界。 \nこんばんは、世界。😀'''
													return input_text, output_text

							def       UpperCAmelCase__		(							self							:       Optional[int]    ,       __magic_name__							:       Optional[Any]				)   ->    str:







													"""simple docstring"""
													UpperCAmelCase_  ,    UpperCAmelCase_     : Optional[Any]         =					self.get_input_output_texts(lowerCamelCase__				)
													UpperCAmelCase_     : int         =					tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__				)
													UpperCAmelCase_     : Tuple         =					tokenizer.decode(lowerCamelCase__    ,       clean_up_tokenization_spaces=lowerCamelCase__				)
													return text, ids

							def       UpperCAmelCase__		(							self							:       Optional[int]				)   ->    int:







													"""simple docstring"""
													pass  # TODO add if relevant

							def       UpperCAmelCase__		(							self							:       Dict				)   ->    Union[str, Any]:







													"""simple docstring"""
													pass  # TODO add if relevant

							def       UpperCAmelCase__		(							self							:       Any				)   ->    Union[str, Any]:







													"""simple docstring"""
													pass  # TODO add if relevant

							def       UpperCAmelCase__		(							self							:       List[Any]				)   ->    Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_     : Tuple         =					self.get_tokenizer()

													# Testing tokenization
													UpperCAmelCase_     : str         =					'''こんにちは、世界。　こんばんは、㔺界。'''
													UpperCAmelCase_     : Dict         =					['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
													UpperCAmelCase_     : Optional[int]         =					tokenizer.tokenize(lowerCamelCase__				)
													self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__				)

													# Testing conversion to ids without special tokens
													UpperCAmelCase_     : str         =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
													UpperCAmelCase_     : Union[str, Any]         =					tokenizer.convert_tokens_to_ids(lowerCamelCase__				)
													self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__				)

													# Testing conversion to ids with special tokens
													UpperCAmelCase_     : List[str]         =					tokens + [tokenizer.unk_token]
													UpperCAmelCase_     : List[str]         =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
													UpperCAmelCase_     : Optional[int]         =					tokenizer.convert_tokens_to_ids(lowerCamelCase__				)
													self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__				)

							@slow
							def       UpperCAmelCase__		(							self							:       Union[str, Any]				)   ->    Any:







													"""simple docstring"""
													UpperCAmelCase_     : Optional[Any]         =					self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''				)

													UpperCAmelCase_     : List[str]         =					tokenizer.encode('''ありがとう。'''    ,       add_special_tokens=lowerCamelCase__				)
													UpperCAmelCase_     : Optional[Any]         =					tokenizer.encode('''どういたしまして。'''    ,       add_special_tokens=lowerCamelCase__				)

													UpperCAmelCase_     : Any         =					tokenizer.build_inputs_with_special_tokens(lowerCamelCase__				)
													UpperCAmelCase_     : List[Any]         =					tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    ,       lowerCamelCase__				)

													assert encoded_sentence == ids_a
													assert encoded_pair == ids_a + ids_a

							def       UpperCAmelCase__		(							self							:       List[Any]				)   ->    List[Any]:







													"""simple docstring"""
													pass



							def       UpperCAmelCase__		(							self							:       str				)   ->    Any:







													"""simple docstring"""
													pass







'''simple docstring'''







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
					import PIL


snake_case_       :    int          =					logging.get_logger(__name__)


class    __a (lowerCamelCase       ):
							__a   :				Optional[Any]       =    ["pixel_values"]

							def __init__(							self							:       str    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Dict[str, int] = None    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Dict[str, int] = None    ,       __magic_name__							:       PILImageResampling = PILImageResampling.BICUBIC    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Union[int, float] = 1 / 2_55    ,       __magic_name__							:       bool = True    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       **__magic_name__							:       List[Any]    ,       )   ->    None:







													"""simple docstring"""
													super().__init__(**__magic_name__				)
													UpperCAmelCase_     : Tuple         =					crop_size if crop_size is not None else {'''height''': 2_56, '''width''': 2_56}
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)
													UpperCAmelCase_     : List[Any]         =					size if size is not None else {'''height''': 2_24, '''width''': 2_24}
													UpperCAmelCase_     : int         =					get_size_dict(__magic_name__				)

													UpperCAmelCase_     : Optional[int]         =					do_center_crop
													UpperCAmelCase_     : List[str]         =					crop_size
													UpperCAmelCase_     : Dict         =					do_resize
													UpperCAmelCase_     : str         =					size
													UpperCAmelCase_     : int         =					resample
													UpperCAmelCase_     : Optional[int]         =					do_rescale
													UpperCAmelCase_     : List[str]         =					rescale_factor
													UpperCAmelCase_     : int         =					do_normalize
													UpperCAmelCase_     : Any         =					image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
													UpperCAmelCase_     : str         =					image_std if image_std is not None else IMAGENET_DEFAULT_STD

							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Dict[str, int]    ,       __magic_name__							:       Optional[int] = None    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Union[str, Any]    ,       )   ->    np.ndarray:







													"""simple docstring"""
													UpperCAmelCase_     : List[Any]         =					self.size if size is None else size
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__				)
													UpperCAmelCase_     : Union[str, Any]         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)

													UpperCAmelCase_  ,    UpperCAmelCase_     : Any         =					get_image_size(__magic_name__				)
													UpperCAmelCase_     : List[str]         =					min(__magic_name__    ,       __magic_name__				)
													UpperCAmelCase_     : Dict         =					(size['''height'''] / crop_size['''height''']) * min_dim
													UpperCAmelCase_     : List[str]         =					(size['''width'''] / crop_size['''width''']) * min_dim
													return center_crop(__magic_name__    ,       size=(cropped_height, cropped_width)    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Union[str, Any]    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Dict[str, int]    ,       __magic_name__							:       PILImageResampling = PIL.Image.BICUBIC    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       str    ,       )   ->    np.ndarray:







													"""simple docstring"""
													UpperCAmelCase_     : str         =					get_size_dict(__magic_name__				)
													if "height" not in size or "width" not in size:
																			raise ValueError(F"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""				)
													return resize(
													    __magic_name__    ,       size=(size['''height'''], size['''width'''])    ,       resample=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Tuple    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Union[int, float]    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Tuple    ,       )   ->    str:







													"""simple docstring"""
													return rescale(__magic_name__    ,       scale=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Optional[int]    ,       __magic_name__							:       np.ndarray    ,       __magic_name__							:       Union[float, List[float]]    ,       __magic_name__							:       Union[float, List[float]]    ,       __magic_name__							:       Optional[Union[str, ChannelDimension]] = None    ,       **__magic_name__							:       Dict    ,       )   ->    np.ndarray:







													"""simple docstring"""
													return normalize(__magic_name__    ,       mean=__magic_name__    ,       std=__magic_name__    ,       data_format=__magic_name__    ,       **__magic_name__				)



							def       UpperCAmelCase__		(							self							:       str    ,       __magic_name__							:       ImageInput    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Dict[str, int]] = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Dict[str, int]] = None    ,       __magic_name__							:       PILImageResampling = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[float] = None    ,       __magic_name__							:       Optional[bool] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[float, List[float]]] = None    ,       __magic_name__							:       Optional[Union[str, TensorType]] = None    ,       __magic_name__							:       ChannelDimension = ChannelDimension.FIRST    ,       **__magic_name__							:       Optional[Any]    ,       )   ->    PIL.Image.Image:







													"""simple docstring"""
													UpperCAmelCase_     : List[str]         =					do_center_crop if do_center_crop is not None else self.do_center_crop
													UpperCAmelCase_     : str         =					crop_size if crop_size is not None else self.crop_size
													UpperCAmelCase_     : int         =					get_size_dict(__magic_name__    ,       param_name='''crop_size'''				)
													UpperCAmelCase_     : Dict         =					do_resize if do_resize is not None else self.do_resize
													UpperCAmelCase_     : Optional[Any]         =					size if size is not None else self.size
													UpperCAmelCase_     : Tuple         =					get_size_dict(__magic_name__				)
													UpperCAmelCase_     : Any         =					resample if resample is not None else self.resample
													UpperCAmelCase_     : int         =					do_rescale if do_rescale is not None else self.do_rescale
													UpperCAmelCase_     : str         =					rescale_factor if rescale_factor is not None else self.rescale_factor
													UpperCAmelCase_     : Optional[Any]         =					do_normalize if do_normalize is not None else self.do_normalize
													UpperCAmelCase_     : Tuple         =					image_mean if image_mean is not None else self.image_mean
													UpperCAmelCase_     : Tuple         =					image_std if image_std is not None else self.image_std

													UpperCAmelCase_     : Optional[Any]         =					make_list_of_images(__magic_name__				)

													if not valid_images(__magic_name__				):
																			raise ValueError(
																			    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																			    '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

													if do_center_crop and crop_size is None:
																			raise ValueError('''If `do_center_crop` is set to `True`, `crop_size` must be provided.'''				)

													if do_resize and size is None:
																			raise ValueError('''Size must be specified if do_resize is True.'''				)

													if do_rescale and rescale_factor is None:
																			raise ValueError('''Rescale factor must be specified if do_rescale is True.'''				)

													if do_normalize and (image_mean is None or image_std is None):
																			raise ValueError('''Image mean and image standard deviation must be specified if do_normalize is True.'''				)

													# All transformations expect numpy arrays.
													UpperCAmelCase_     : Tuple         =					[to_numpy_array(__magic_name__				) for image in images]

													if do_center_crop:
																			UpperCAmelCase_     : int         =					[self.center_crop(__magic_name__    ,       __magic_name__    ,       size=__magic_name__				) for image in images]

													if do_resize:
																			UpperCAmelCase_     : Dict         =					[self.resize(image=__magic_name__    ,       size=__magic_name__    ,       resample=__magic_name__				) for image in images]

													if do_rescale:
																			UpperCAmelCase_     : Union[str, Any]         =					[self.rescale(image=__magic_name__    ,       scale=__magic_name__				) for image in images]

													if do_normalize:
																			UpperCAmelCase_     : Any         =					[self.normalize(image=__magic_name__    ,       mean=__magic_name__    ,       std=__magic_name__				) for image in images]

													UpperCAmelCase_     : Optional[int]         =					[to_channel_dimension_format(__magic_name__    ,       __magic_name__				) for image in images]

													UpperCAmelCase_     : Union[str, Any]         =					{'''pixel_values''': images}
													return BatchFeature(data=__magic_name__    ,       tensor_type=__magic_name__				)





