





import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_lowerCamelCase							=     16
_lowerCamelCase							=     32






def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							List[Any]      ):
		return int(x / 2**20      )




class 					UpperCamelCase_    :


							def __enter__(							self      :Dict						)				->				Any:



									"""simple docstring"""



									gc.collect()
									torch.cuda.empty_cache()
									torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
									SCREAMING_SNAKE_CASE__								= torch.cuda.memory_allocated()
									SCREAMING_SNAKE_CASE__								= psutil.Process()

									SCREAMING_SNAKE_CASE__								= self.cpu_mem_used()
									SCREAMING_SNAKE_CASE__								= True
									SCREAMING_SNAKE_CASE__								= threading.Thread(target=self.peak_monitor_func						)
									SCREAMING_SNAKE_CASE__								= True
									peak_monitor_thread.start()
									return self


							def        _snake_case					(							self      :Optional[int]						)				->				Tuple:



									"""simple docstring"""



									return self.process.memory_info().rss


							def        _snake_case					(							self      :List[str]						)				->				List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= -1

									while True:
											SCREAMING_SNAKE_CASE__								= max(self.cpu_mem_used()							,			self.cpu_peak						)

											# can't sleep or will not catch the peak right (this comment is here on purpose)
											# time.sleep(0.001) # 1msec

											if not self.peak_monitoring:
													break


							def __exit__(							self      :int							,			*__A      :Tuple						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= False

									gc.collect()
									torch.cuda.empty_cache()
									SCREAMING_SNAKE_CASE__								= torch.cuda.memory_allocated()
									SCREAMING_SNAKE_CASE__								= torch.cuda.max_memory_allocated()
									SCREAMING_SNAKE_CASE__								= bamb(self.end - self.begin						)
									SCREAMING_SNAKE_CASE__								= bamb(self.peak - self.begin						)

									SCREAMING_SNAKE_CASE__								= self.cpu_mem_used()
									SCREAMING_SNAKE_CASE__								= bamb(self.cpu_end - self.cpu_begin						)
									SCREAMING_SNAKE_CASE__								= bamb(self.cpu_peak - self.cpu_begin						)
									# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
	from accelerate.test_utils.training import mocked_dataloaders

	_lowerCamelCase							=     mocked_dataloaders  # noqa: F811






def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							int   ,				UpperCamelCase__:							int      ):
		# For testing only
		if os.environ.get("""TESTING_MOCKED_DATALOADERS"""   ,				UpperCamelCase__      ) == "1":
				SCREAMING_SNAKE_CASE__								= 2

		# New Code #
		# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
		SCREAMING_SNAKE_CASE__								= FullyShardedDataParallelPlugin(
		    state_dict_config=FullStateDictConfig(offload_to_cpu=UpperCamelCase__   ,				ranka_only=UpperCamelCase__      )   ,				optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=UpperCamelCase__   ,				ranka_only=UpperCamelCase__      )   ,				)

		# Initialize accelerator
		if args.with_tracking:
				SCREAMING_SNAKE_CASE__								= Accelerator(
				    cpu=args.cpu   ,				mixed_precision=args.mixed_precision   ,				log_with="""wandb"""   ,				project_dir=args.logging_dir   ,				fsdp_plugin=UpperCamelCase__   ,				)
		else:
				SCREAMING_SNAKE_CASE__								= Accelerator(fsdp_plugin=UpperCamelCase__      )
		accelerator.print(accelerator.distributed_type      )

		if hasattr(args.checkpointing_steps   ,				"""isdigit"""      ):
				if args.checkpointing_steps == "epoch":
						SCREAMING_SNAKE_CASE__								= args.checkpointing_steps
				elif args.checkpointing_steps.isdigit():
						SCREAMING_SNAKE_CASE__								= int(args.checkpointing_steps      )
				else:
						raise ValueError(
						    f'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'''      )
		else:
				SCREAMING_SNAKE_CASE__								= None
		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		SCREAMING_SNAKE_CASE__								= config["""lr"""]
		SCREAMING_SNAKE_CASE__								= int(config["""num_epochs"""]      )
		SCREAMING_SNAKE_CASE__								= int(config["""seed"""]      )
		SCREAMING_SNAKE_CASE__								= int(config["""batch_size"""]      )

		# We need to initialize the trackers we use, and also store our configuration
		if args.with_tracking:
				SCREAMING_SNAKE_CASE__								= vars(UpperCamelCase__      )
				accelerator.init_trackers("""fsdp_glue_no_trainer"""   ,				UpperCamelCase__      )

		SCREAMING_SNAKE_CASE__								= AutoTokenizer.from_pretrained(args.model_name_or_path      )
		SCREAMING_SNAKE_CASE__								= load_dataset("""glue"""   ,				"""mrpc"""      )
		SCREAMING_SNAKE_CASE__								= evaluate.load("""glue"""   ,				"""mrpc"""      )

		def tokenize_function(UpperCamelCase__:							List[Any]      ):
				# max_length=None => use the model max length (it's actually the default)
				SCREAMING_SNAKE_CASE__								= tokenizer(examples["""sentence1"""]   ,				examples["""sentence2"""]   ,				truncation=UpperCamelCase__   ,				max_length=UpperCamelCase__      )
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		# starting with the main process first:
		with accelerator.main_process_first():
				SCREAMING_SNAKE_CASE__								= datasets.map(
				    UpperCamelCase__   ,				batched=UpperCamelCase__   ,				remove_columns=["""idx""", """sentence1""", """sentence2"""]   ,				)

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		SCREAMING_SNAKE_CASE__								= tokenized_datasets.rename_column("""label"""   ,				"""labels"""      )

		# If the batch size is too big we use gradient accumulation
		SCREAMING_SNAKE_CASE__								= 1
		if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
				SCREAMING_SNAKE_CASE__								= batch_size // MAX_GPU_BATCH_SIZE
				SCREAMING_SNAKE_CASE__								= MAX_GPU_BATCH_SIZE

		def collate_fn(UpperCamelCase__:							Dict      ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				SCREAMING_SNAKE_CASE__								= 128 if accelerator.distributed_type == DistributedType.TPU else None
				# When using mixed precision we want round multiples of 8/16
				if accelerator.mixed_precision == "fp8":
						SCREAMING_SNAKE_CASE__								= 16
				elif accelerator.mixed_precision != "no":
						SCREAMING_SNAKE_CASE__								= 8
				else:
						SCREAMING_SNAKE_CASE__								= None

				return tokenizer.pad(
				    UpperCamelCase__   ,				padding="""longest"""   ,				max_length=UpperCamelCase__   ,				pad_to_multiple_of=UpperCamelCase__   ,				return_tensors="""pt"""   ,				)

		# Instantiate dataloaders.
		SCREAMING_SNAKE_CASE__								= DataLoader(
		    tokenized_datasets["""train"""]   ,				shuffle=UpperCamelCase__   ,				collate_fn=UpperCamelCase__   ,				batch_size=UpperCamelCase__      )
		SCREAMING_SNAKE_CASE__								= DataLoader(
		    tokenized_datasets["""validation"""]   ,				shuffle=UpperCamelCase__   ,				collate_fn=UpperCamelCase__   ,				batch_size=UpperCamelCase__      )

		set_seed(UpperCamelCase__      )

		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		SCREAMING_SNAKE_CASE__								= AutoModelForSequenceClassification.from_pretrained(
		    args.model_name_or_path   ,				return_dict=UpperCamelCase__   ,				low_cpu_mem_usage=UpperCamelCase__      )

		# New Code #
		# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
		SCREAMING_SNAKE_CASE__								= accelerator.prepare(UpperCamelCase__      )
		accelerator.print(UpperCamelCase__      )

		# Instantiate optimizer
		# New Code #
		# For FSDP feature, at present it doesn't support multiple parameter groups,
		# so we need to create a single parameter group for the whole model
		SCREAMING_SNAKE_CASE__								= torch.optim.AdamW(params=model.parameters()   ,				lr=UpperCamelCase__   ,				weight_decay=2e-4      )

		# Instantiate scheduler
		SCREAMING_SNAKE_CASE__								= get_linear_schedule_with_warmup(
		    optimizer=UpperCamelCase__   ,				num_warmup_steps=10   ,				num_training_steps=(len(UpperCamelCase__      ) * num_epochs) // gradient_accumulation_steps   ,				)

		# New Code #
		# For FSDP feature, prepare everything except the model as we have already prepared the model
		# before creating the optimizer
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.
		SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= accelerator.prepare(
		    UpperCamelCase__   ,				UpperCamelCase__   ,				UpperCamelCase__   ,				UpperCamelCase__      )

		SCREAMING_SNAKE_CASE__								= 0

		# Potentially load in the weights and states from a previous save
		if args.resume_from_checkpoint:
				if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
						accelerator.print(f'''Resumed from checkpoint: {args.resume_from_checkpoint}'''      )
						accelerator.load_state(args.resume_from_checkpoint      )
						SCREAMING_SNAKE_CASE__								= os.path.basename(args.resume_from_checkpoint      )
				else:
						# Get the most recent checkpoint
						SCREAMING_SNAKE_CASE__								= [f.name for f in os.scandir(os.getcwd()      ) if f.is_dir()]
						dirs.sort(key=os.path.getctime      )
						SCREAMING_SNAKE_CASE__								= dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
				# Extract `epoch_{i}` or `step_{i}`
				SCREAMING_SNAKE_CASE__								= os.path.splitext(UpperCamelCase__      )[0]

				if "epoch" in training_difference:
						num_epochs -= int(training_difference.replace("""epoch_"""   ,				""""""      )      )
						SCREAMING_SNAKE_CASE__								= None
				else:
						SCREAMING_SNAKE_CASE__								= int(training_difference.replace("""step_"""   ,				""""""      )      )
						num_epochs -= resume_step // len(UpperCamelCase__      )
						# If resuming by step, we also need to know exactly how far into the DataLoader we went
						SCREAMING_SNAKE_CASE__								= (num_epochs * len(UpperCamelCase__      )) - resume_step

    # Now we train the model
		for epoch in range(UpperCamelCase__      ):
				# New Code #
				# context manager to track the peak memory usage during the training epoch
				with TorchTracemalloc() as tracemalloc:
						model.train()
						if args.with_tracking:
								SCREAMING_SNAKE_CASE__								= 0
						for step, batch in enumerate(UpperCamelCase__      ):
								# We need to skip steps until we reach the resumed step
								if args.resume_from_checkpoint and epoch == 0:
										if resume_step is not None and step < resume_step:
												pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device      )
								SCREAMING_SNAKE_CASE__								= model(**UpperCamelCase__      )
								SCREAMING_SNAKE_CASE__								= outputs.loss
								SCREAMING_SNAKE_CASE__								= loss / gradient_accumulation_steps
								# We keep track of the loss at each epoch
								if args.with_tracking:
										total_loss += loss.detach().float()
								accelerator.backward(UpperCamelCase__      )
								if step % gradient_accumulation_steps == 0:
										optimizer.step()
										lr_scheduler.step()
										optimizer.zero_grad()
										# accelerator.print(lr_scheduler.get_lr())

								overall_step += 1

								if isinstance(UpperCamelCase__   ,				UpperCamelCase__      ):
										SCREAMING_SNAKE_CASE__								= f'''step_{overall_step}'''
										if overall_step % checkpointing_steps == 0:
												if args.output_dir is not None:
														SCREAMING_SNAKE_CASE__								= os.path.join(args.output_dir   ,				UpperCamelCase__      )
												accelerator.save_state(UpperCamelCase__      )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
				accelerator.print("""Memory before entering the train : {}""".format(bamb(tracemalloc.begin      )      )      )
				accelerator.print("""Memory consumed at the end of the train (end-begin): {}""".format(tracemalloc.used      )      )
				accelerator.print("""Peak Memory consumed during the train (max-begin): {}""".format(tracemalloc.peaked      )      )
				accelerator.print(
				    """Total Peak Memory consumed during the train (max): {}""".format(
				        tracemalloc.peaked + bamb(tracemalloc.begin      )      )      )
				# Logging the peak memory usage of the GPU to the tracker
				if args.with_tracking:
						accelerator.log(
						    {
						        """train_total_peak_memory""": tracemalloc.peaked + bamb(tracemalloc.begin      ),
						    }   ,				step=UpperCamelCase__   ,				)

				# New Code #
				# context manager to track the peak memory usage during the evaluation
				with TorchTracemalloc() as tracemalloc:
						model.eval()
						for step, batch in enumerate(UpperCamelCase__      ):
								# We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device      )
								with torch.no_grad():
										SCREAMING_SNAKE_CASE__								= model(**UpperCamelCase__      )
								SCREAMING_SNAKE_CASE__								= outputs.logits.argmax(dim=-1      )
								SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= accelerator.gather_for_metrics((predictions, batch["""labels"""])      )
								metric.add_batch(
								    predictions=UpperCamelCase__   ,				references=UpperCamelCase__   ,				)

						SCREAMING_SNAKE_CASE__								= metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(f'''epoch {epoch}:'''   ,				UpperCamelCase__      )
						if args.with_tracking:
								accelerator.log(
								    {
								        """accuracy""": eval_metric["""accuracy"""],
								        """f1""": eval_metric["""f1"""],
								        """train_loss""": total_loss.item() / len(UpperCamelCase__      ),
								    }   ,				step=UpperCamelCase__   ,				)

						if checkpointing_steps == "epoch":
								SCREAMING_SNAKE_CASE__								= f'''epoch_{epoch}'''
								if args.output_dir is not None:
										SCREAMING_SNAKE_CASE__								= os.path.join(args.output_dir   ,				UpperCamelCase__      )
								accelerator.save_state(UpperCamelCase__      )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
				accelerator.print("""Memory before entering the eval : {}""".format(bamb(tracemalloc.begin      )      )      )
				accelerator.print("""Memory consumed at the end of the eval (end-begin): {}""".format(tracemalloc.used      )      )
				accelerator.print("""Peak Memory consumed during the eval (max-begin): {}""".format(tracemalloc.peaked      )      )
				accelerator.print(
				    """Total Peak Memory consumed during the eval (max): {}""".format(tracemalloc.peaked + bamb(tracemalloc.begin      )      )      )
				# Logging the peak memory usage of the GPU to the tracker
				if args.with_tracking:
						accelerator.log(
						    {
						        """eval_total_peak_memory""": tracemalloc.peaked + bamb(tracemalloc.begin      ),
						    }   ,				step=UpperCamelCase__   ,				)

		if args.with_tracking:
				accelerator.end_training()






def 			SCREAMING_SNAKE_CASE__							(    ):
		SCREAMING_SNAKE_CASE__								= argparse.ArgumentParser(description="""Simple example of training script."""      )
		parser.add_argument(
		    """--mixed_precision"""   ,				type=UpperCamelCase__   ,				default=UpperCamelCase__   ,				choices=["""no""", """fp16""", """bf16""", """fp8"""]   ,				help="""Whether to use mixed precision. Choose"""
		    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
		    """and an Nvidia Ampere GPU."""   ,				)
		parser.add_argument("""--cpu"""   ,				action="""store_true"""   ,				help="""If passed, will train on the CPU."""      )
		parser.add_argument(
		    """--checkpointing_steps"""   ,				type=UpperCamelCase__   ,				default=UpperCamelCase__   ,				help="""Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch."""   ,				)
		parser.add_argument(
		    """--resume_from_checkpoint"""   ,				type=UpperCamelCase__   ,				default=UpperCamelCase__   ,				help="""If the training should continue from a checkpoint folder."""   ,				)
		parser.add_argument(
		    """--with_tracking"""   ,				action="""store_true"""   ,				help="""Whether to load in all available experiment trackers from the environment and use them for logging."""   ,				)
		parser.add_argument(
		    """--output_dir"""   ,				type=UpperCamelCase__   ,				default="""."""   ,				help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""   ,				)
		parser.add_argument(
		    """--logging_dir"""   ,				type=UpperCamelCase__   ,				default="""logs"""   ,				help="""Location on where to store experiment tracking logs`"""   ,				)
		parser.add_argument(
		    """--model_name_or_path"""   ,				type=UpperCamelCase__   ,				help="""Path to pretrained model or model identifier from huggingface.co/models."""   ,				required=UpperCamelCase__   ,				)
		SCREAMING_SNAKE_CASE__								= parser.parse_args()
		SCREAMING_SNAKE_CASE__								= {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
		training_function(UpperCamelCase__   ,				UpperCamelCase__      )


if __name__ == "__main__":
	main()






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					UpperCamelCase_    (       unittest.TestCase      ):


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
											SCREAMING_SNAKE_CASE__								= tokenizer_class.from_pretrained(__A						)
											self.assertIsNotNone(__A						)
											self.assertIsInstance(__A							,			__A						)
											self.assertIsInstance(__A							,			__A						)

											for special_tok in tokenizer.all_special_tokens:
													self.assertIsInstance(__A							,			__A						)
													SCREAMING_SNAKE_CASE__								= tokenizer.convert_tokens_to_ids(__A						)
													self.assertIsInstance(__A							,			__A						)


							def        _snake_case					(							self      :List[str]							,			__A      :BatchEncoding							,			__A      :Optional[Callable] = None						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pickle.dumps(__A						)
									self.assertIsNotNone(__A						)

									SCREAMING_SNAKE_CASE__								= pickle.loads(__A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast							,			be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
											self.assertIsNotNone(be_restored.encodings						)
									else:
											self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()							,			be_restored.values()						):
											if equal_op:
													self.assertTrue(equal_op(__A							,			__A						)						)
											else:
													self.assertEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__A						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									self.assertEqual(TensorType("""tf"""						)							,			TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)							,			TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)							,			TensorType.NUMPY						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				List[Any]:



									"""simple docstring"""



									import numpy as np

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)


							@require_tf
							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[Any]:



									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__A      :Union[str, Any]							,			__A      :Any						):
											return tf.reduce_all(tf.equal(__A							,			__A						)						)

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)


							@require_torch
							@require_tokenizers
							def        _snake_case					(							self      :List[Any]						)				->				int:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
											self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
											self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)


							@require_tokenizers
							def        _snake_case					(							self      :List[str]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= tokenizer_r(["""Test""", """\xad""", """test"""]							,			is_split_into_words=__A						)

									self.assertEqual(encoded.word_to_tokens(0						)							,			TokenSpan(start=1							,			end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)							,			__A						)
									self.assertEqual(encoded.word_to_tokens(2						)							,			TokenSpan(start=2							,			end=3						)						)


							def        _snake_case					(							self      :Tuple						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_tf
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_flax
							def        _snake_case					(							self      :Dict						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							def        _snake_case					(							self      :List[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				Dict:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tf
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									import tensorflow as tf

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tokenizers
							def        _snake_case					(							self      :Tuple						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=__A						)


							@require_tokenizers
							def        _snake_case					(							self      :Any						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
											bert_tokenizer.save(os.path.join(__A							,			"""tokenizer.json"""						)						)
											PreTrainedTokenizerFast(tokenizer_file=os.path.join(__A							,			"""tokenizer.json"""						)						)