





import torch

from diffusers import DDIMScheduler

from .test_schedulers import SchedulerCommonTest




class 					UpperCamelCase_    (       A__      ):
							lowerCamelCase_          =   (DDIMScheduler,)
							lowerCamelCase_          =   (("eta", 0.0), ("num_inference_steps", 50))


							def        _snake_case					(							self      :Union[str, Any]							,			**__A      :Optional[Any]						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= {
									    '''num_train_timesteps''': 1000,
									    '''beta_start''': 0.0_0_0_1,
									    '''beta_end''': 0.0_2,
									    '''beta_schedule''': '''linear''',
									    '''clip_sample''': True,
									}

									config.update(**__snake_case						)
									return config


							def        _snake_case					(							self      :str							,			**__A      :Any						)				->				List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.scheduler_classes[0]
									SCREAMING_SNAKE_CASE__								= self.get_scheduler_config(**__snake_case						)
									SCREAMING_SNAKE_CASE__								= scheduler_class(**__snake_case						)

									SCREAMING_SNAKE_CASE__								= 10, 0.0

									SCREAMING_SNAKE_CASE__								= self.dummy_model()
									SCREAMING_SNAKE_CASE__								= self.dummy_sample_deter

									scheduler.set_timesteps(__snake_case						)

									for t in scheduler.timesteps:
											SCREAMING_SNAKE_CASE__								= model(__snake_case							,			__snake_case						)
											SCREAMING_SNAKE_CASE__								= scheduler.step(__snake_case							,			__snake_case							,			__snake_case							,			__snake_case						).prev_sample

									return sample


							def        _snake_case					(							self      :Tuple						)				->				Optional[Any]:



									"""simple docstring"""



									for timesteps in [100, 500, 1000]:
											self.check_over_configs(num_train_timesteps=__snake_case						)


							def        _snake_case					(							self      :Tuple						)				->				List[str]:



									"""simple docstring"""



									for steps_offset in [0, 1]:
											self.check_over_configs(steps_offset=__snake_case						)

									SCREAMING_SNAKE_CASE__								= self.scheduler_classes[0]
									SCREAMING_SNAKE_CASE__								= self.get_scheduler_config(steps_offset=1						)
									SCREAMING_SNAKE_CASE__								= scheduler_class(**__snake_case						)
									scheduler.set_timesteps(5						)
									assert torch.equal(scheduler.timesteps							,			torch.LongTensor([801, 601, 401, 201, 1]						)						)


							def        _snake_case					(							self      :Union[str, Any]						)				->				Any:



									"""simple docstring"""



									for beta_start, beta_end in zip([0.0_0_0_1, 0.0_0_1, 0.0_1, 0.1]							,			[0.0_0_2, 0.0_2, 0.2, 2]						):
											self.check_over_configs(beta_start=__snake_case							,			beta_end=__snake_case						)


							def        _snake_case					(							self      :Any						)				->				str:



									"""simple docstring"""



									for schedule in ["linear", "squaredcos_cap_v2"]:
											self.check_over_configs(beta_schedule=__snake_case						)


							def        _snake_case					(							self      :Tuple						)				->				Optional[int]:



									"""simple docstring"""



									for prediction_type in ["epsilon", "v_prediction"]:
											self.check_over_configs(prediction_type=__snake_case						)


							def        _snake_case					(							self      :Optional[int]						)				->				Optional[Any]:



									"""simple docstring"""



									for clip_sample in [True, False]:
											self.check_over_configs(clip_sample=__snake_case						)


							def        _snake_case					(							self      :Optional[int]						)				->				Dict:



									"""simple docstring"""



									for timestep_spacing in ["trailing", "leading"]:
											self.check_over_configs(timestep_spacing=__snake_case						)


							def        _snake_case					(							self      :List[str]						)				->				Any:



									"""simple docstring"""



									for rescale_betas_zero_snr in [True, False]:
											self.check_over_configs(rescale_betas_zero_snr=__snake_case						)


							def        _snake_case					(							self      :str						)				->				List[Any]:



									"""simple docstring"""



									self.check_over_configs(thresholding=__snake_case						)
									for threshold in [0.5, 1.0, 2.0]:
											for prediction_type in ["epsilon", "v_prediction"]:
													self.check_over_configs(
													    thresholding=__snake_case							,			prediction_type=__snake_case							,			sample_max_value=__snake_case							,			)


							def        _snake_case					(							self      :Optional[int]						)				->				Tuple:



									"""simple docstring"""



									for t in [1, 10, 49]:
											self.check_over_forward(time_step=__snake_case						)


							def        _snake_case					(							self      :Any						)				->				int:



									"""simple docstring"""



									for t, num_inference_steps in zip([1, 10, 50]							,			[10, 50, 500]						):
											self.check_over_forward(time_step=__snake_case							,			num_inference_steps=__snake_case						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									for t, eta in zip([1, 10, 49]							,			[0.0, 0.5, 1.0]						):
											self.check_over_forward(time_step=__snake_case							,			eta=__snake_case						)


							def        _snake_case					(							self      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.scheduler_classes[0]
									SCREAMING_SNAKE_CASE__								= self.get_scheduler_config()
									SCREAMING_SNAKE_CASE__								= scheduler_class(**__snake_case						)

									assert torch.sum(torch.abs(scheduler._get_variance(0							,			0						) - 0.0						)						) < 1E-5
									assert torch.sum(torch.abs(scheduler._get_variance(420							,			400						) - 0.1_4_7_7_1						)						) < 1E-5
									assert torch.sum(torch.abs(scheduler._get_variance(980							,			960						) - 0.3_2_4_6_0						)						) < 1E-5
									assert torch.sum(torch.abs(scheduler._get_variance(0							,			0						) - 0.0						)						) < 1E-5
									assert torch.sum(torch.abs(scheduler._get_variance(487							,			486						) - 0.0_0_9_7_9						)						) < 1E-5
									assert torch.sum(torch.abs(scheduler._get_variance(999							,			998						) - 0.0_2						)						) < 1E-5


							def        _snake_case					(							self      :Any						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.full_loop()

									SCREAMING_SNAKE_CASE__								= torch.sum(torch.abs(__snake_case						)						)
									SCREAMING_SNAKE_CASE__								= torch.mean(torch.abs(__snake_case						)						)

									assert abs(result_sum.item() - 172.0067						) < 1E-2
									assert abs(result_mean.item() - 0.2_2_3_9_6_7						) < 1E-3


							def        _snake_case					(							self      :List[str]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.full_loop(prediction_type="""v_prediction"""						)

									SCREAMING_SNAKE_CASE__								= torch.sum(torch.abs(__snake_case						)						)
									SCREAMING_SNAKE_CASE__								= torch.mean(torch.abs(__snake_case						)						)

									assert abs(result_sum.item() - 5_2.5_3_0_2						) < 1E-2
									assert abs(result_mean.item() - 0.0_6_8_4						) < 1E-3


							def        _snake_case					(							self      :Union[str, Any]						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.full_loop(set_alpha_to_one=__snake_case							,			beta_start=0.0_1						)
									SCREAMING_SNAKE_CASE__								= torch.sum(torch.abs(__snake_case						)						)
									SCREAMING_SNAKE_CASE__								= torch.mean(torch.abs(__snake_case						)						)

									assert abs(result_sum.item() - 149.8295						) < 1E-2
									assert abs(result_mean.item() - 0.1_9_5_1						) < 1E-3


							def        _snake_case					(							self      :Union[str, Any]						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.full_loop(set_alpha_to_one=__snake_case							,			beta_start=0.0_1						)
									SCREAMING_SNAKE_CASE__								= torch.sum(torch.abs(__snake_case						)						)
									SCREAMING_SNAKE_CASE__								= torch.mean(torch.abs(__snake_case						)						)

									assert abs(result_sum.item() - 149.0784						) < 1E-2
									assert abs(result_mean.item() - 0.1_9_4_1						) < 1E-3






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					UpperCamelCase_    (       unittest.TestCase      ):


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
											SCREAMING_SNAKE_CASE__								= tokenizer_class.from_pretrained(__A						)
											self.assertIsNotNone(__A						)
											self.assertIsInstance(__A							,			__A						)
											self.assertIsInstance(__A							,			__A						)

											for special_tok in tokenizer.all_special_tokens:
													self.assertIsInstance(__A							,			__A						)
													SCREAMING_SNAKE_CASE__								= tokenizer.convert_tokens_to_ids(__A						)
													self.assertIsInstance(__A							,			__A						)


							def        _snake_case					(							self      :List[str]							,			__A      :BatchEncoding							,			__A      :Optional[Callable] = None						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pickle.dumps(__A						)
									self.assertIsNotNone(__A						)

									SCREAMING_SNAKE_CASE__								= pickle.loads(__A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast							,			be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
											self.assertIsNotNone(be_restored.encodings						)
									else:
											self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()							,			be_restored.values()						):
											if equal_op:
													self.assertTrue(equal_op(__A							,			__A						)						)
											else:
													self.assertEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__A						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									self.assertEqual(TensorType("""tf"""						)							,			TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)							,			TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)							,			TensorType.NUMPY						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				List[Any]:



									"""simple docstring"""



									import numpy as np

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)


							@require_tf
							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[Any]:



									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__A      :Union[str, Any]							,			__A      :Any						):
											return tf.reduce_all(tf.equal(__A							,			__A						)						)

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)


							@require_torch
							@require_tokenizers
							def        _snake_case					(							self      :List[Any]						)				->				int:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
											self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
											self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)


							@require_tokenizers
							def        _snake_case					(							self      :List[str]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= tokenizer_r(["""Test""", """\xad""", """test"""]							,			is_split_into_words=__A						)

									self.assertEqual(encoded.word_to_tokens(0						)							,			TokenSpan(start=1							,			end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)							,			__A						)
									self.assertEqual(encoded.word_to_tokens(2						)							,			TokenSpan(start=2							,			end=3						)						)


							def        _snake_case					(							self      :Tuple						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_tf
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_flax
							def        _snake_case					(							self      :Dict						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							def        _snake_case					(							self      :List[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				Dict:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tf
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									import tensorflow as tf

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tokenizers
							def        _snake_case					(							self      :Tuple						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=__A						)


							@require_tokenizers
							def        _snake_case					(							self      :Any						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
											bert_tokenizer.save(os.path.join(__A							,			"""tokenizer.json"""						)						)
											PreTrainedTokenizerFast(tokenizer_file=os.path.join(__A							,			"""tokenizer.json"""						)						)