





import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 						_A( snake_case__    ,			unittest.TestCase       ):


				"""simple docstring"""


				UpperCamelCase   :  int    =     GPTNeoXJapaneseTokenizer
				UpperCamelCase   :  List[str]    =     False
				UpperCamelCase   :  Optional[int]    =     {'''do_clean_text''': False, '''add_prefix_space''': False}



				def  UpperCAmelCase_		(      self					):
									super().setUp()

									__A     :						str          =    [
									    'こん',
									    'こんに',
									    'にちは',
									    'ばんは',
									    '世界,㔺界',
									    '、',
									    '。',
									    '<BR>',
									    '<SP>',
									    '<TAB>',
									    '<URL>',
									    '<EMAIL>',
									    '<TEL>',
									    '<DATE>',
									    '<PRICE>',
									    '<BLOCK>',
									    '<KIGOU>',
									    '<U2000U2BFF>',
									    '<|emoji1|>',
									    '<unk>',
									    '<|startoftext|>',
									    '<|endoftext|>',
									]
									__A     :						Optional[int]          =    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
									__A     :						Tuple          =    {'unk_token': '<unk>'}

									__A     :						Optional[Any]          =    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES['vocab_file']					)
									__A     :						str          =    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES['emoji_file']					)
									with open(self.vocab_file					, 'w'					, encoding='utf-8'					) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)
									with open(self.emoji_file					, 'w'					) as emoji_writer:
														emoji_writer.write(json.dumps(_A					)					)



				def  UpperCAmelCase_		(      self					, **_A					):
									kwargs.update(self.special_tokens_map					)
									return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname					, **_A					)



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Optional[Any]          =    'こんにちは、世界。 \nこんばんは、㔺界。😀'
									__A     :						Optional[int]          =    'こんにちは、世界。 \nこんばんは、世界。😀'
									return input_text, output_text



				def  UpperCAmelCase_		(      self					, _A					):
									__A   ,    __A     :						int          =    self.get_input_output_texts(_A					)
									__A     :						int          =    tokenizer.encode(_A					, add_special_tokens=_A					)
									__A     :						Optional[Any]          =    tokenizer.decode(_A					, clean_up_tokenization_spaces=_A					)
									return text, ids



				def  UpperCAmelCase_		(      self					):
									pass  # TODO add if relevant



				def  UpperCAmelCase_		(      self					):
									pass  # TODO add if relevant



				def  UpperCAmelCase_		(      self					):
									pass  # TODO add if relevant



				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.get_tokenizer()

									# Testing tokenization
									__A     :						Dict          =    'こんにちは、世界。　こんばんは、㔺界。'
									__A     :						List[Any]          =    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
									__A     :						List[Any]          =    tokenizer.tokenize(_A					)
									self.assertListEqual(_A					, _A					)

									# Testing conversion to ids without special tokens
									__A     :						Any          =    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
									__A     :						Dict          =    tokenizer.convert_tokens_to_ids(_A					)
									self.assertListEqual(_A					, _A					)

									# Testing conversion to ids with special tokens
									__A     :						Dict          =    tokens + [tokenizer.unk_token]
									__A     :						Dict          =    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
									__A     :						Optional[int]          =    tokenizer.convert_tokens_to_ids(_A					)
									self.assertListEqual(_A					, _A					)



				@slow
				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.tokenizer_class.from_pretrained('abeja/gpt-neox-japanese-2.7b'					)

									__A     :						List[Any]          =    tokenizer.encode('ありがとう。'					, add_special_tokens=_A					)
									__A     :						Dict          =    tokenizer.encode('どういたしまして。'					, add_special_tokens=_A					)

									__A     :						int          =    tokenizer.build_inputs_with_special_tokens(_A					)
									__A     :						Tuple          =    tokenizer.build_inputs_with_special_tokens(_A					, _A					)

									assert encoded_sentence == ids_a
									assert encoded_pair == ids_a + ids_a



				def  UpperCAmelCase_		(      self					):
									# Intentionally convert some words to accommodate character fluctuations unique to Japanese
									pass





				def  UpperCAmelCase_		(      self					):
									# tokenizer has no padding token
									pass











import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import TvltImageProcessor







def 			_SCREAMING_SNAKE_CASE    (       a ,						a=10 ,						a=10 ,						a=False ,						a=False  )				->				Union[str, Any]:
					__A     :						str          =    []
					for i in range(image_processor_tester.num_frames  ):
										video.append(np.random.randint(2_55 ,						size=(image_processor_tester.num_channels, width, height) ,						dtype=np.uinta  )  )

					if not numpify and not torchify:
										# PIL expects the channel dimension as last dimension
										__A     :						Union[str, Any]          =    [Image.fromarray(np.moveaxis(a ,						0 ,						-1  )  ) for frame in video]

					if torchify:
										__A     :						Dict          =    [torch.from_numpy(a  ) for frame in video]

					return video







def 			_SCREAMING_SNAKE_CASE    (       a ,						a=False ,						a=False ,						a=False  )				->				Union[str, Any]:
					assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

					__A     :						List[Any]          =    []
					for i in range(image_processor_tester.batch_size  ):
										if equal_resolution:
															__A     :						int          =    image_processor_tester.max_resolution
										else:
															__A   ,    __A     :						Tuple          =    np.random.choice(
															    np.arange(image_processor_tester.min_resolution ,						image_processor_tester.max_resolution  ) ,						2  )
															__A     :						Optional[int]          =    prepare_video(
															    image_processor_tester=a ,						width=a ,						height=a ,						numpify=a ,						torchify=a ,						)
										video_inputs.append(a  )

					return video_inputs





class 						_A( unittest.TestCase       ):


				"""simple docstring"""


				def __init__(      self					, _A					, _A=7					, _A=3					, _A=4					, _A=18					, _A=30					, _A=400					, _A=True					, _A=None					, _A=True					, _A=[0.5, 0.5, 0.5]					, _A=[0.5, 0.5, 0.5]					, _A=True					, _A=None					, ):
									__A     :						List[str]          =    size if size is not None else {'shortest_edge': 18}
									__A     :						Tuple          =    crop_size if crop_size is not None else {'height': 18, 'width': 18}

									__A     :						Dict          =    parent
									__A     :						str          =    batch_size
									__A     :						int          =    num_channels
									__A     :						Optional[Any]          =    num_frames
									__A     :						Any          =    image_size
									__A     :						List[str]          =    min_resolution
									__A     :						List[Any]          =    max_resolution
									__A     :						Tuple          =    do_resize
									__A     :						List[str]          =    size
									__A     :						str          =    do_normalize
									__A     :						str          =    image_mean
									__A     :						int          =    image_std
									__A     :						Optional[Any]          =    do_center_crop
									__A     :						Union[str, Any]          =    crop_size





				def  UpperCAmelCase_		(      self					):
									return {
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_normalize": self.do_normalize,
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_center_crop": self.do_center_crop,
									    "crop_size": self.crop_size,
									}





@require_torch
@require_vision
class 						_A( snake_case__    ,			unittest.TestCase       ):


				"""simple docstring"""


				UpperCamelCase   :  Optional[Any]    =     TvltImageProcessor if is_vision_available() else None



				def  UpperCAmelCase_		(      self					):
									__A     :						str          =    TvltImageProcessorTester(self					)



				@property
				def  UpperCAmelCase_		(      self					):
									return self.image_processor_tester.prepare_image_processor_dict()



				def  UpperCAmelCase_		(      self					):
									__A     :						Union[str, Any]          =    self.image_processing_class(**self.image_processor_dict					)
									self.assertTrue(hasattr(_A					, 'image_mean'					)					)
									self.assertTrue(hasattr(_A					, 'image_std'					)					)
									self.assertTrue(hasattr(_A					, 'do_normalize'					)					)
									self.assertTrue(hasattr(_A					, 'do_resize'					)					)
									self.assertTrue(hasattr(_A					, 'do_center_crop'					)					)
									self.assertTrue(hasattr(_A					, 'size'					)					)



				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						List[str]          =    self.image_processing_class(**self.image_processor_dict					)
									# create random PIL videos
									__A     :						List[str]          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, Image.Image					)

									# Test not batched input
									__A     :						str          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						List[Any]          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )



				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						Any          =    self.image_processing_class(**self.image_processor_dict					)
									# create random numpy tensors
									__A     :						Dict          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					, numpify=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, np.ndarray					)

									# Test not batched input
									__A     :						Dict          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						Optional[int]          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )





				def  UpperCAmelCase_		(      self					):
									# Initialize image_processor
									__A     :						Any          =    self.image_processing_class(**self.image_processor_dict					)
									# create random PyTorch tensors
									__A     :						Dict          =    prepare_video_inputs(self.image_processor_tester					, equal_resolution=_A					, torchify=_A					)
									for video in video_inputs:
														self.assertIsInstance(_A					, _A					)
														self.assertIsInstance(video[0]					, torch.Tensor					)

									# Test not batched input
									__A     :						Optional[Any]          =    image_processor(video_inputs[0]					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )

									# Test batched
									__A     :						int          =    image_processor(_A					, return_tensors='pt'					).pixel_values
									self.assertEqual(
									    encoded_videos.shape					, (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )					, )




