




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    snake_case__							(_lowerCAmelCase   ,	unittest.TestCase       ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_	:							str										= ElectraTokenizer
				SCREAMING_SNAKE_CASE_	:							List[Any]										= ElectraTokenizerFast
				SCREAMING_SNAKE_CASE_	:							Union[str, Any]										= True
				SCREAMING_SNAKE_CASE_	:							List[str]										= True
				SCREAMING_SNAKE_CASE_	:							Optional[Any]										= filter_non_english





				def 							__UpperCAmelCase							(				self    :					Optional[Any]							) ->					List[Any]:
							super().setUp()

							a   =      [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							a   =      os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES["vocab_file"]							)
							with open(self.vocab_file ,				"w" ,				encoding="utf-8"							) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)





				def 							__UpperCAmelCase							(				self    :					Tuple ,				__lowerCamelCase    :					int							) ->					Optional[int]:
							a   =      "UNwant\u00E9d,running"
							a   =      "unwanted, running"
							return input_text, output_text





				def 							__UpperCAmelCase							(				self    :					List[str]							) ->					str:
							a   =      self.tokenizer_class(self.vocab_file							)

							a   =      tokenizer.tokenize("UNwant\u00E9d,running"							)
							self.assertListEqual(_lowerCamelCase ,				["un", "##want", "##ed", ",", "runn", "##ing"]							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							) ,				[9, 6, 7, 12, 10, 11]							)





				def 							__UpperCAmelCase							(				self    :					List[Any]							) ->					Tuple:
							if not self.test_rust_tokenizer:
										return

							a   =      self.get_tokenizer()
							a   =      self.get_rust_tokenizer()

							a   =      "UNwant\u00E9d,running"

							a   =      tokenizer.tokenize(_lowerCamelCase							)
							a   =      rust_tokenizer.tokenize(_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

							a   =      tokenizer.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
							a   =      rust_tokenizer.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

							a   =      self.get_rust_tokenizer()
							a   =      tokenizer.encode(_lowerCamelCase							)
							a   =      rust_tokenizer.encode(_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

							# With lower casing
							a   =      self.get_tokenizer(do_lower_case=_lowerCamelCase							)
							a   =      self.get_rust_tokenizer(do_lower_case=_lowerCamelCase							)

							a   =      "UNwant\u00E9d,running"

							a   =      tokenizer.tokenize(_lowerCamelCase							)
							a   =      rust_tokenizer.tokenize(_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

							a   =      tokenizer.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
							a   =      rust_tokenizer.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

							a   =      self.get_rust_tokenizer()
							a   =      tokenizer.encode(_lowerCamelCase							)
							a   =      rust_tokenizer.encode(_lowerCamelCase							)
							self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)





				def 							__UpperCAmelCase							(				self    :					Optional[Any]							) ->					Dict:
							a   =      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"							) ,				["ah", "\u535A", "\u63A8", "zz"]							)





				def 							__UpperCAmelCase							(				self    :					int							) ->					List[Any]:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "							) ,				["hello", "!", "how", "are", "you", "?"]							)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							) ,				["hello"]							)





				def 							__UpperCAmelCase							(				self    :					List[str]							) ->					Dict:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase ,				strip_accents=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							) ,				["hällo", "!", "how", "are", "you", "?"]							)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							) ,				["h\u00E9llo"]							)





				def 							__UpperCAmelCase							(				self    :					Union[str, Any]							) ->					Any:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase ,				strip_accents=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							) ,				["hallo", "!", "how", "are", "you", "?"]							)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							) ,				["hello"]							)





				def 							__UpperCAmelCase							(				self    :					int							) ->					Optional[Any]:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							) ,				["hallo", "!", "how", "are", "you", "?"]							)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							) ,				["hello"]							)





				def 							__UpperCAmelCase							(				self    :					Tuple							) ->					Optional[Any]:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "							) ,				["HeLLo", "!", "how", "Are", "yoU", "?"]							)





				def 							__UpperCAmelCase							(				self    :					Union[str, Any]							) ->					Optional[Any]:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase ,				strip_accents=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							) ,				["HäLLo", "!", "how", "Are", "yoU", "?"]							)





				def 							__UpperCAmelCase							(				self    :					Union[str, Any]							) ->					Any:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase ,				strip_accents=_lowerCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							) ,				["HaLLo", "!", "how", "Are", "yoU", "?"]							)





				def 							__UpperCAmelCase							(				self    :					Dict							) ->					Union[str, Any]:
							a   =      BasicTokenizer(do_lower_case=_lowerCamelCase ,				never_split=["[UNK]"]							)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"							) ,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]							)





				def 							__UpperCAmelCase							(				self    :					int							) ->					Optional[int]:
							a   =      ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							a   =      {}
							for i, token in enumerate(_lowerCamelCase							):
										a   =      i
							a   =      WordpieceTokenizer(vocab=_lowerCamelCase ,				unk_token="[UNK]"							)

							self.assertListEqual(tokenizer.tokenize(""							) ,				[]							)

							self.assertListEqual(tokenizer.tokenize("unwanted running"							) ,				["un", "##want", "##ed", "runn", "##ing"]							)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"							) ,				["[UNK]", "runn", "##ing"]							)





				def 							__UpperCAmelCase							(				self    :					int							) ->					Any:
							self.assertTrue(_is_whitespace(" "							)							)
							self.assertTrue(_is_whitespace("\t"							)							)
							self.assertTrue(_is_whitespace("\r"							)							)
							self.assertTrue(_is_whitespace("\n"							)							)
							self.assertTrue(_is_whitespace("\u00A0"							)							)

							self.assertFalse(_is_whitespace("A"							)							)
							self.assertFalse(_is_whitespace("-"							)							)





				def 							__UpperCAmelCase							(				self    :					Any							) ->					int:
							self.assertTrue(_is_control("\u0005"							)							)

							self.assertFalse(_is_control("A"							)							)
							self.assertFalse(_is_control(" "							)							)
							self.assertFalse(_is_control("\t"							)							)
							self.assertFalse(_is_control("\r"							)							)





				def 							__UpperCAmelCase							(				self    :					Optional[int]							) ->					Union[str, Any]:
							self.assertTrue(_is_punctuation("-"							)							)
							self.assertTrue(_is_punctuation("$"							)							)
							self.assertTrue(_is_punctuation("`"							)							)
							self.assertTrue(_is_punctuation("."							)							)

							self.assertFalse(_is_punctuation("A"							)							)
							self.assertFalse(_is_punctuation(" "							)							)





				def 							__UpperCAmelCase							(				self    :					Optional[int]							) ->					Dict:
							a   =      self.get_tokenizer()
							a   =      self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_lowerCamelCase							) for t in ["Test", "\xad", "test"]] ,				[["[UNK]"], [], ["[UNK]"]]							)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(_lowerCamelCase							) for t in ["Test", "\xad", "test"]] ,				[["[UNK]"], [], ["[UNK]"]]							)





				@slow
				def 							__UpperCAmelCase							(				self    :					Optional[int]							) ->					Optional[Any]:
							a   =      self.tokenizer_class.from_pretrained("google/electra-base-discriminator"							)

							a   =      tokenizer.encode("sequence builders" ,				add_special_tokens=_lowerCamelCase							)
							a   =      tokenizer.encode("multi-sequence build" ,				add_special_tokens=_lowerCamelCase							)

							a   =      tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							)
							a   =      tokenizer.build_inputs_with_special_tokens(_lowerCamelCase ,				_lowerCamelCase							)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]





				def 							__UpperCAmelCase							(				self    :					Tuple							) ->					Tuple:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
													a   =      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,				**_lowerCamelCase							)

													a   =      f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													a   =      tokenizer_r.encode_plus(
													    _lowerCamelCase ,				return_attention_mask=_lowerCamelCase ,				return_token_type_ids=_lowerCamelCase ,				return_offsets_mapping=_lowerCamelCase ,				add_special_tokens=_lowerCamelCase ,				)

													a   =      tokenizer_r.do_lower_case if hasattr(_lowerCamelCase ,				"do_lower_case"							) else False
													a   =      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results] ,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]							)							)
													self.assertEqual([e[0] for e in expected_results] ,				tokens["offset_mapping"]							)





				def 							__UpperCAmelCase							(				self    :					Optional[int]							) ->					Any:
							a   =      ["的", "人", "有"]
							a   =      "".join(_lowerCamelCase							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
													a   =      True
													a   =      self.tokenizer_class.from_pretrained(_lowerCamelCase ,				**_lowerCamelCase							)
													a   =      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,				**_lowerCamelCase							)

													a   =      tokenizer_p.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
													a   =      tokenizer_r.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)

													a   =      tokenizer_r.convert_ids_to_tokens(_lowerCamelCase							)
													a   =      tokenizer_p.convert_ids_to_tokens(_lowerCamelCase							)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)
													self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)

													a   =      False
													a   =      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,				**_lowerCamelCase							)
													a   =      self.tokenizer_class.from_pretrained(_lowerCamelCase ,				**_lowerCamelCase							)

													a   =      tokenizer_r.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)
													a   =      tokenizer_p.encode(_lowerCamelCase ,				add_special_tokens=_lowerCamelCase							)

													a   =      tokenizer_r.convert_ids_to_tokens(_lowerCamelCase							)
													a   =      tokenizer_p.convert_ids_to_tokens(_lowerCamelCase							)

													# it is expected that only the first Chinese character is not preceded by "##".
													a   =      [
													    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase							)
													]
													self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)
													self.assertListEqual(_lowerCamelCase ,				_lowerCamelCase							)












import torch
import torch.nn.functional as F

from diffusers import VQDiffusionScheduler

from .test_schedulers import SchedulerCommonTest




class    snake_case__							(_UpperCamelCase       ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_	:							Dict										= (VQDiffusionScheduler,)





				def 							__UpperCAmelCase							(				self    :					Tuple ,				**__lowerCamelCase    :					Optional[int]							) ->					Optional[Any]:
							a   =      {
							    "num_vec_classes": 40_97,
							    "num_train_timesteps": 1_00,
							}

							config.update(**__lowerCamelCase							)
							return config





				def 							__UpperCAmelCase							(				self    :					List[str] ,				__lowerCamelCase    :					Any							) ->					Union[str, Any]:
							a   =      4
							a   =      8
							a   =      8

							a   =      torch.randint(0 ,				__lowerCamelCase ,				(batch_size, height * width)							)

							return sample





				@property
				def 							__UpperCAmelCase							(				self    :					List[str]							) ->					Optional[Any]:
							assert False





				def 							__UpperCAmelCase							(				self    :					Any ,				__lowerCamelCase    :					Optional[int]							) ->					int:
							def model(__lowerCamelCase    :					Tuple ,				__lowerCamelCase    :					Any ,				*__lowerCamelCase    :					Optional[Any]							):
										a   ,		a   =      sample.shape
										a   =      torch.rand((batch_size, num_vec_classes - 1, num_latent_pixels)							)
										a   =      F.log_softmax(logits.double() ,				dim=1							).float()
										return return_value

							return model





				def 							__UpperCAmelCase							(				self    :					Optional[Any]							) ->					Any:
							for timesteps in [2, 5, 1_00, 10_00]:
										self.check_over_configs(num_train_timesteps=__lowerCamelCase							)





				def 							__UpperCAmelCase							(				self    :					Any							) ->					List[Any]:
							for num_vec_classes in [5, 1_00, 10_00, 40_00]:
										self.check_over_configs(num_vec_classes=__lowerCamelCase							)





				def 							__UpperCAmelCase							(				self    :					Optional[Any]							) ->					List[Any]:
							for t in [0, 50, 99]:
										self.check_over_forward(time_step=__lowerCamelCase							)





				def 							__UpperCAmelCase							(				self    :					List[Any]							) ->					Optional[Any]:
							pass






