






import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.semantic_stable_diffusion import SemanticStableDiffusionPipeline as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()


class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											# clean up the VRAM after each test
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()
							@property
							def lowerCAmelCase_	(       self: Any  )						->						str:
											snake_case__													=       1
											snake_case__													=       3
											snake_case__													=       (32, 32)

											snake_case__													=       floats_tensor((batch_size, num_channels) + sizes    ,		rng=random.Random(0  )  ).to(_UpperCAmelCase  )
											return image
							@property
							def lowerCAmelCase_	(       self: Dict  )						->						Any:
											torch.manual_seed(0  )
											snake_case__													=       UNetaDConditionModel(
											    block_out_channels=(32, 64)    ,		layers_per_block=2    ,		sample_size=32    ,		in_channels=4    ,		out_channels=4    ,		down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')    ,		up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')    ,		cross_attention_dim=32    ,		)
											return model
							@property
							def lowerCAmelCase_	(       self: int  )						->						Union[str, Any]:
											torch.manual_seed(0  )
											snake_case__													=       AutoencoderKL(
											    block_out_channels=[32, 64]    ,		in_channels=3    ,		out_channels=3    ,		down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']    ,		up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']    ,		latent_channels=4    ,		)
											return model
							@property
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Optional[int]:
											torch.manual_seed(0  )
											snake_case__													=       CLIPTextConfig(
											    bos_token_id=0    ,		eos_token_id=2    ,		hidden_size=32    ,		intermediate_size=37    ,		layer_norm_eps=1e-05    ,		num_attention_heads=4    ,		num_hidden_layers=5    ,		pad_token_id=1    ,		vocab_size=10_00    ,		)
											return CLIPTextModel(_UpperCAmelCase  )
							@property
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Union[str, Any]:
											def extract(*UpperCamelCase: Optional[int]    ,		**UpperCamelCase: Union[str, Any]  ):


															class 					__SCREAMING_SNAKE_CASE:
																						def __init__(       self: Tuple  )						->						List[Any]:
																										snake_case__													=       torch.ones([0]  )


																						def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: Optional[int]  )						->						Tuple:
																										self.pixel_values.to(_UpperCAmelCase  )
																										return self


															return Out()

											return extract
							def lowerCAmelCase_	(       self: Tuple  )						->						Tuple:
											snake_case__													=       "cpu"  # ensure determinism for the device-dependent torch.Generator
											snake_case__													=       self.dummy_cond_unet
											snake_case__													=       DDIMScheduler(
											    beta_start=0.00_085    ,		beta_end=0.012    ,		beta_schedule='scaled_linear'    ,		clip_sample=_UpperCAmelCase    ,		set_alpha_to_one=_UpperCAmelCase    ,		)

											snake_case__													=       self.dummy_vae
											snake_case__													=       self.dummy_text_encoder
											snake_case__													=       CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'  )

											# make sure here that pndm scheduler skips prk
											snake_case__													=       StableDiffusionPipeline(
											    unet=_UpperCAmelCase    ,		scheduler=_UpperCAmelCase    ,		vae=_UpperCAmelCase    ,		text_encoder=_UpperCAmelCase    ,		tokenizer=_UpperCAmelCase    ,		safety_checker=_UpperCAmelCase    ,		feature_extractor=self.dummy_extractor    ,		)
											snake_case__													=       sd_pipe.to(_UpperCAmelCase  )
											sd_pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "A painting of a squirrel eating a burger"

											snake_case__													=       torch.Generator(device=_UpperCAmelCase  ).manual_seed(0  )
											snake_case__													=       sd_pipe([prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=6.0    ,		num_inference_steps=2    ,		output_type='np'  )
											snake_case__													=       output.images

											snake_case__													=       torch.Generator(device=_UpperCAmelCase  ).manual_seed(0  )
											snake_case__													=       sd_pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=6.0    ,		num_inference_steps=2    ,		output_type='np'    ,		return_dict=_UpperCAmelCase    ,		)[0]

											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       image_from_tuple[0, -3:, -3:, -1]

											assert image.shape == (1, 64, 64, 3)
											snake_case__													=       np.array([0.5_753, 0.6_114, 0.5_001, 0.5_034, 0.5_470, 0.4_729, 0.4_971, 0.4_867, 0.4_867]  )

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2
											assert np.abs(image_from_tuple_slice.flatten() - expected_slice  ).max() < 1e-2
							def lowerCAmelCase_	(       self: int  )						->						Tuple:
											snake_case__													=       "cpu"  # ensure determinism for the device-dependent torch.Generator
											snake_case__													=       self.dummy_cond_unet
											snake_case__													=       PNDMScheduler(skip_prk_steps=_UpperCAmelCase  )
											snake_case__													=       self.dummy_vae
											snake_case__													=       self.dummy_text_encoder
											snake_case__													=       CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'  )

											# make sure here that pndm scheduler skips prk
											snake_case__													=       StableDiffusionPipeline(
											    unet=_UpperCAmelCase    ,		scheduler=_UpperCAmelCase    ,		vae=_UpperCAmelCase    ,		text_encoder=_UpperCAmelCase    ,		tokenizer=_UpperCAmelCase    ,		safety_checker=_UpperCAmelCase    ,		feature_extractor=self.dummy_extractor    ,		)
											snake_case__													=       sd_pipe.to(_UpperCAmelCase  )
											sd_pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "A painting of a squirrel eating a burger"
											snake_case__													=       torch.Generator(device=_UpperCAmelCase  ).manual_seed(0  )
											snake_case__													=       sd_pipe([prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=6.0    ,		num_inference_steps=2    ,		output_type='np'  )

											snake_case__													=       output.images

											snake_case__													=       torch.Generator(device=_UpperCAmelCase  ).manual_seed(0  )
											snake_case__													=       sd_pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=6.0    ,		num_inference_steps=2    ,		output_type='np'    ,		return_dict=_UpperCAmelCase    ,		)[0]

											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       image_from_tuple[0, -3:, -3:, -1]

											assert image.shape == (1, 64, 64, 3)
											snake_case__													=       np.array([0.5_122, 0.5_712, 0.4_825, 0.5_053, 0.5_646, 0.4_769, 0.5_179, 0.4_894, 0.4_994]  )

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2
											assert np.abs(image_from_tuple_slice.flatten() - expected_slice  ).max() < 1e-2
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Any:
											snake_case__													=       StableDiffusionPipeline.from_pretrained(
											    'hf-internal-testing/tiny-stable-diffusion-lms-pipe'    ,		safety_checker=_UpperCAmelCase  )
											assert isinstance(_UpperCAmelCase    ,		_UpperCAmelCase  )
											assert isinstance(pipe.scheduler    ,		_UpperCAmelCase  )
											assert pipe.safety_checker is None

											snake_case__													=       pipe('example prompt'    ,		num_inference_steps=2  ).images[0]
											assert image is not None

											# check that there's no error when saving a pipeline with one of the models being None
											with tempfile.TemporaryDirectory() as tmpdirname:
															pipe.save_pretrained(_UpperCAmelCase  )
															snake_case__													=       StableDiffusionPipeline.from_pretrained(_UpperCAmelCase  )

											# sanity check that the pipeline still works
											assert pipe.safety_checker is None
											snake_case__													=       pipe('example prompt'    ,		num_inference_steps=2  ).images[0]
											assert image is not None


							@unittest.skipIf(torch_device != 'cuda'    ,		'This test requires a GPU'  )
							def lowerCAmelCase_	(       self: Any  )						->						Optional[int]:
											snake_case__													=       self.dummy_cond_unet
											snake_case__													=       PNDMScheduler(skip_prk_steps=_UpperCAmelCase  )
											snake_case__													=       self.dummy_vae
											snake_case__													=       self.dummy_text_encoder
											snake_case__													=       CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'  )

											# put models in fp16
											snake_case__													=       unet.half()
											snake_case__													=       vae.half()
											snake_case__													=       bert.half()

											# make sure here that pndm scheduler skips prk
											snake_case__													=       StableDiffusionPipeline(
											    unet=_UpperCAmelCase    ,		scheduler=_UpperCAmelCase    ,		vae=_UpperCAmelCase    ,		text_encoder=_UpperCAmelCase    ,		tokenizer=_UpperCAmelCase    ,		safety_checker=_UpperCAmelCase    ,		feature_extractor=self.dummy_extractor    ,		)
											snake_case__													=       sd_pipe.to(_UpperCAmelCase  )
											sd_pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "A painting of a squirrel eating a burger"
											snake_case__													=       sd_pipe([prompt]    ,		num_inference_steps=2    ,		output_type='np'  ).images

											assert image.shape == (1, 64, 64, 3)


@nightly
@require_torch_gpu
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: Union[str, Any]  )						->						List[Any]:
											# clean up the VRAM after each test
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()
							def lowerCAmelCase_	(       self: Any  )						->						List[Any]:
											snake_case__													=       "cuda"
											snake_case__													=       StableDiffusionPipeline.from_pretrained('runwayml/stable-diffusion-v1-5'  )
											snake_case__													=       pipe.to(_UpperCAmelCase  )
											pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "a photo of a cat"
											snake_case__													=       {
											    "editing_prompt": ["sunglasses"],
											    "reverse_editing_direction": [False],
											    "edit_warmup_steps": 10,
											    "edit_guidance_scale": 6,
											    "edit_threshold": 0.95,
											    "edit_momentum_scale": 0.5,
											    "edit_mom_beta": 0.6,
											}

											snake_case__													=       3
											snake_case__													=       7

											# no sega enabled
											snake_case__													=       torch.Generator(_UpperCAmelCase  )
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.34_673_113,
											    0.38_492_733,
											    0.37_597_352,
											    0.34_086_335,
											    0.35_650_748,
											    0.35_579_205,
											    0.3_384_763,
											    0.34_340_236,
											    0.3_573_271,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

											# with sega enabled
											# generator = torch.manual_seed(seed)
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		**_UpperCAmelCase    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.41_887_826,
											    0.37_728_766,
											    0.30_138_272,
											    0.41_416_335,
											    0.41_664_985,
											    0.36_283_392,
											    0.36_191_246,
											    0.43_364_465,
											    0.43_001_732,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2
							def lowerCAmelCase_	(       self: Dict  )						->						Tuple:
											snake_case__													=       "cuda"
											snake_case__													=       StableDiffusionPipeline.from_pretrained('runwayml/stable-diffusion-v1-5'  )
											snake_case__													=       pipe.to(_UpperCAmelCase  )
											pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "an image of a crowded boulevard, realistic, 4k"
											snake_case__													=       {
											    "editing_prompt": "crowd, crowded, people",
											    "reverse_editing_direction": True,
											    "edit_warmup_steps": 10,
											    "edit_guidance_scale": 8.3,
											    "edit_threshold": 0.9,
											    "edit_momentum_scale": 0.5,
											    "edit_mom_beta": 0.6,
											}

											snake_case__													=       9
											snake_case__													=       7

											# no sega enabled
											snake_case__													=       torch.Generator(_UpperCAmelCase  )
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.43_497_998,
											    0.91_814_065,
											    0.7_540_739,
											    0.55_580_205,
											    0.8_467_265,
											    0.5_389_691,
											    0.62_574_506,
											    0.58_897_763,
											    0.50_926_757,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

											# with sega enabled
											# generator = torch.manual_seed(seed)
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		**_UpperCAmelCase    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.3_089_719,
											    0.30_500_144,
											    0.29_016_042,
											    0.30_630_964,
											    0.325_687,
											    0.29_419_225,
											    0.2_908_091,
											    0.28_723_598,
											    0.27_696_294,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2
							def lowerCAmelCase_	(       self: Optional[int]  )						->						List[str]:
											snake_case__													=       "cuda"
											snake_case__													=       StableDiffusionPipeline.from_pretrained('runwayml/stable-diffusion-v1-5'  )
											snake_case__													=       pipe.to(_UpperCAmelCase  )
											pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "a castle next to a river"
											snake_case__													=       {
											    "editing_prompt": ["boat on a river, boat", "monet, impression, sunrise"],
											    "reverse_editing_direction": False,
											    "edit_warmup_steps": [15, 18],
											    "edit_guidance_scale": 6,
											    "edit_threshold": [0.9, 0.8],
											    "edit_momentum_scale": 0.5,
											    "edit_mom_beta": 0.6,
											}

											snake_case__													=       48
											snake_case__													=       7

											# no sega enabled
											snake_case__													=       torch.Generator(_UpperCAmelCase  )
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.75_163_555,
											    0.76_037_145,
											    0.61_785,
											    0.9_189_673,
											    0.8_627_701,
											    0.85_189_694,
											    0.8_512_813,
											    0.87_012_076,
											    0.8_312_857,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

											# with sega enabled
											# generator = torch.manual_seed(seed)
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		**_UpperCAmelCase    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.73_553_365,
											    0.7_537_271,
											    0.74_341_905,
											    0.66_480_356,
											    0.6_472_925,
											    0.63_039_416,
											    0.64_812_905,
											    0.6_749_717,
											    0.6_517_102,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2


							def lowerCAmelCase_	(       self: List[Any]  )						->						Any:
											snake_case__													=       "cuda"
											snake_case__													=       StableDiffusionPipeline.from_pretrained('runwayml/stable-diffusion-v1-5'    ,		torch_dtype=torch.floataa  )
											snake_case__													=       pipe.to(_UpperCAmelCase  )
											pipe.set_progress_bar_config(disable=_UpperCAmelCase  )

											snake_case__													=       "a photo of a cat"
											snake_case__													=       {
											    "editing_prompt": ["sunglasses"],
											    "reverse_editing_direction": [False],
											    "edit_warmup_steps": 10,
											    "edit_guidance_scale": 6,
											    "edit_threshold": 0.95,
											    "edit_momentum_scale": 0.5,
											    "edit_mom_beta": 0.6,
											}

											snake_case__													=       3
											snake_case__													=       7

											# no sega enabled
											snake_case__													=       torch.Generator(_UpperCAmelCase  )
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.34_887_695,
											    0.3_876_953,
											    0.375,
											    0.34_423_828,
											    0.3_581_543,
											    0.35_717_773,
											    0.3_383_789,
											    0.34_570_312,
											    0.359_375,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

											# with sega enabled
											# generator = torch.manual_seed(seed)
											generator.manual_seed(_UpperCAmelCase  )
											snake_case__													=       pipe(
											    [prompt]    ,		generator=_UpperCAmelCase    ,		guidance_scale=_UpperCAmelCase    ,		num_inference_steps=50    ,		output_type='np'    ,		width=5_12    ,		height=5_12    ,		**_UpperCAmelCase    ,		)

											snake_case__													=       output.images
											snake_case__													=       image[0, -3:, -3:, -1]
											snake_case__													=       [
											    0.42_285_156,
											    0.36_914_062,
											    0.29_077_148,
											    0.42_041_016,
											    0.41_918_945,
											    0.35_498_047,
											    0.3_618_164,
											    0.4_423_828,
											    0.43_115_234,
											]

											assert image.shape == (1, 5_12, 5_12, 3)

											assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2












import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    ElectraTokenizer
							_UpperCAmelCase	      =    ElectraTokenizerFast
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    filter_non_english
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											super().setUp()

											snake_case__													=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case__													=       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: List[str]  )						->						Union[str, Any]:
											snake_case__													=       'UNwant\u00E9d,running'
											snake_case__													=       'unwanted, running'
											return input_text, output_text
							def lowerCAmelCase_	(       self: List[str]  )						->						List[Any]:
											snake_case__													=       self.tokenizer_class(self.vocab_file  )

											snake_case__													=       tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(UpperCamelCase    ,		['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase  )    ,		[9, 6, 7, 12, 10, 11]  )
							def lowerCAmelCase_	(       self: str  )						->						Tuple:
											if not self.test_rust_tokenizer:
															return

											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											# With lower casing
											snake_case__													=       self.get_tokenizer(do_lower_case=UpperCamelCase  )
											snake_case__													=       self.get_rust_tokenizer(do_lower_case=UpperCamelCase  )

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											snake_case__													=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,		['ah', '\u535A', '\u63A8', 'zz']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[Any]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['h\u00E9llo']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: str  )						->						List[str]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
							def lowerCAmelCase_	(       self: Any  )						->						Dict:
											snake_case__													=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											snake_case__													=       {}
											for i, token in enumerate(UpperCamelCase  ):
															snake_case__													=       i
											snake_case__													=       WordpieceTokenizer(vocab=UpperCamelCase    ,		unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )    ,		[]  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,		['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,		['[UNK]', 'runn', '##ing']  )
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Tuple:
											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
							def lowerCAmelCase_	(       self: str  )						->						Any:
											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )
							@slow
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											snake_case__													=       tokenizer.encode('sequence builders'    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       tokenizer.encode('multi-sequence build'    ,		add_special_tokens=UpperCamelCase  )

											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase  )
											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase    ,		UpperCamelCase  )

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			snake_case__													=       tokenizer_r.encode_plus(
																			    UpperCamelCase    ,		return_attention_mask=UpperCamelCase    ,		return_token_type_ids=UpperCamelCase    ,		return_offsets_mapping=UpperCamelCase    ,		add_special_tokens=UpperCamelCase    ,		)

																			snake_case__													=       tokenizer_r.do_lower_case if hasattr(UpperCamelCase    ,		'do_lower_case'  ) else False
																			snake_case__													=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																			self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping']  )


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       ['的', '人', '有']
											snake_case__													=       ''.join(UpperCamelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       True
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

																			snake_case__													=       False
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case__													=       [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase  )
																			]
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )




