

import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCAmelCase_	     =     16
lowerCAmelCase_	     =     32


def   __SCREAMING_SNAKE_CASE (SCREAMING_SNAKE_CASE__	):
						return int(x / 2**20	)





class      snake_case_       :







			'''simple docstring'''

			def __enter__(					self			:    Union[str, Any]							)				->Optional[Any]:
									gc.collect()
									torch.cuda.empty_cache()
									torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
									snake_case_       =     torch.cuda.memory_allocated()
									snake_case_       =     psutil.Process()

									snake_case_       =     self.cpu_mem_used()
									snake_case_       =     True
									snake_case_       =     threading.Thread(target=self.peak_monitor_func							)
									snake_case_       =     True
									peak_monitor_thread.start()
									return self







			def     snake_case__(					self			:    Union[str, Any]							)				->Dict:
									return self.process.memory_info().rss







			def     snake_case__(					self			:    str							)				->Optional[Any]:
									snake_case_       =     -1

									while True:
															snake_case_       =     max(self.cpu_mem_used()							,       self.cpu_peak							)

															# can't sleep or will not catch the peak right (this comment is here on purpose)
															# time.sleep(0.001) # 1msec

															if not self.peak_monitoring:
																					break







			def __exit__(					self			:    Dict							,       *_UpperCamelCase			:    Optional[Any]							)				->int:
									snake_case_       =     False

									gc.collect()
									torch.cuda.empty_cache()
									snake_case_       =     torch.cuda.memory_allocated()
									snake_case_       =     torch.cuda.max_memory_allocated()
									snake_case_       =     bamb(self.end - self.begin							)
									snake_case_       =     bamb(self.peak - self.begin							)

									snake_case_       =     self.cpu_mem_used()
									snake_case_       =     bamb(self.cpu_end - self.cpu_begin							)
									snake_case_       =     bamb(self.cpu_peak - self.cpu_begin							)
									# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
					from accelerate.test_utils.training import mocked_dataloaders

					lowerCAmelCase_	     =     mocked_dataloaders  # noqa: F811


def   __SCREAMING_SNAKE_CASE (SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__	):
						# For testing only
						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''  ,		SCREAMING_SNAKE_CASE__	) == "1":
												snake_case_       =     2

						# New Code #
						# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
						snake_case_       =     FullyShardedDataParallelPlugin(
						    state_dict_config=FullStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,		ranka_only=SCREAMING_SNAKE_CASE__	)  ,		optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,		ranka_only=SCREAMING_SNAKE_CASE__	)  ,		)

						# Initialize accelerator
						if args.with_tracking:
												snake_case_       =     Accelerator(
												    cpu=args.cpu  ,		mixed_precision=args.mixed_precision  ,		log_with='''wandb'''  ,		project_dir=args.logging_dir  ,		fsdp_plugin=SCREAMING_SNAKE_CASE__  ,		)
						else:
												snake_case_       =     Accelerator(fsdp_plugin=SCREAMING_SNAKE_CASE__	)
						accelerator.print(accelerator.distributed_type	)

						if hasattr(args.checkpointing_steps  ,		'''isdigit'''	):
												if args.checkpointing_steps == "epoch":
																		snake_case_       =     args.checkpointing_steps
												elif args.checkpointing_steps.isdigit():
																		snake_case_       =     int(args.checkpointing_steps	)
												else:
																		raise ValueError(
																		    F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'''	)
						else:
												snake_case_       =     None
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						snake_case_       =     config['''lr''']
						snake_case_       =     int(config['''num_epochs''']	)
						snake_case_       =     int(config['''seed''']	)
						snake_case_       =     int(config['''batch_size''']	)

						# We need to initialize the trackers we use, and also store our configuration
						if args.with_tracking:
												snake_case_       =     vars(SCREAMING_SNAKE_CASE__	)
												accelerator.init_trackers('''fsdp_glue_no_trainer'''  ,		SCREAMING_SNAKE_CASE__	)

						snake_case_       =     AutoTokenizer.from_pretrained(args.model_name_or_path	)
						snake_case_       =     load_dataset('''glue'''  ,		'''mrpc'''	)
						snake_case_       =     evaluate.load('''glue'''  ,		'''mrpc'''	)

						def tokenize_function(SCREAMING_SNAKE_CASE__	):
												# max_length=None => use the model max length (it's actually the default)
												snake_case_       =     tokenizer(examples['''sentence1''']  ,		examples['''sentence2''']  ,		truncation=SCREAMING_SNAKE_CASE__  ,		max_length=SCREAMING_SNAKE_CASE__	)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												snake_case_       =     datasets.map(
												    SCREAMING_SNAKE_CASE__  ,		batched=SCREAMING_SNAKE_CASE__  ,		remove_columns=['''idx''', '''sentence1''', '''sentence2''']  ,		)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						snake_case_       =     tokenized_datasets.rename_column('''label'''  ,		'''labels'''	)

						# If the batch size is too big we use gradient accumulation
						snake_case_       =     1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												snake_case_       =     batch_size // MAX_GPU_BATCH_SIZE
												snake_case_       =     MAX_GPU_BATCH_SIZE

						def collate_fn(SCREAMING_SNAKE_CASE__	):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												snake_case_       =     128 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		snake_case_       =     16
												elif accelerator.mixed_precision != "no":
																		snake_case_       =     8
												else:
																		snake_case_       =     None

												return tokenizer.pad(
												    SCREAMING_SNAKE_CASE__  ,		padding='''longest'''  ,		max_length=SCREAMING_SNAKE_CASE__  ,		pad_to_multiple_of=SCREAMING_SNAKE_CASE__  ,		return_tensors='''pt'''  ,		)

						# Instantiate dataloaders.
						snake_case_       =     DataLoader(
						    tokenized_datasets['''train''']  ,		shuffle=SCREAMING_SNAKE_CASE__  ,		collate_fn=SCREAMING_SNAKE_CASE__  ,		batch_size=SCREAMING_SNAKE_CASE__	)
						snake_case_       =     DataLoader(
						    tokenized_datasets['''validation''']  ,		shuffle=SCREAMING_SNAKE_CASE__  ,		collate_fn=SCREAMING_SNAKE_CASE__  ,		batch_size=SCREAMING_SNAKE_CASE__	)

						set_seed(SCREAMING_SNAKE_CASE__	)

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						snake_case_       =     AutoModelForSequenceClassification.from_pretrained(
						    args.model_name_or_path  ,		return_dict=SCREAMING_SNAKE_CASE__  ,		low_cpu_mem_usage=SCREAMING_SNAKE_CASE__	)

						# New Code #
						# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
						snake_case_       =     accelerator.prepare(SCREAMING_SNAKE_CASE__	)
						accelerator.print(SCREAMING_SNAKE_CASE__	)

						# Instantiate optimizer
						# New Code #
						# For FSDP feature, at present it doesn't support multiple parameter groups,
						# so we need to create a single parameter group for the whole model
						snake_case_       =     torch.optim.AdamW(params=model.parameters()  ,		lr=SCREAMING_SNAKE_CASE__  ,		weight_decay=2E-4	)

						# Instantiate scheduler
						snake_case_       =     get_linear_schedule_with_warmup(
						    optimizer=SCREAMING_SNAKE_CASE__  ,		num_warmup_steps=10  ,		num_training_steps=(len(SCREAMING_SNAKE_CASE__	) * num_epochs) // gradient_accumulation_steps  ,		)

						# New Code #
						# For FSDP feature, prepare everything except the model as we have already prepared the model
						# before creating the optimizer
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						snake_case_,   snake_case_,   snake_case_,   snake_case_       =     accelerator.prepare(
						    SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__	)

						snake_case_       =     0

						# Potentially load in the weights and states from a previous save
						if args.resume_from_checkpoint:
												if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
																		accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}'''	)
																		accelerator.load_state(args.resume_from_checkpoint	)
																		snake_case_       =     os.path.basename(args.resume_from_checkpoint	)
												else:
																		# Get the most recent checkpoint
																		snake_case_       =     [f.name for f in os.scandir(os.getcwd()	) if f.is_dir()]
																		dirs.sort(key=os.path.getctime	)
																		snake_case_       =     dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
												# Extract `epoch_{i}` or `step_{i}`
												snake_case_       =     os.path.splitext(SCREAMING_SNAKE_CASE__	)[0]

												if "epoch" in training_difference:
																		num_epochs -= int(training_difference.replace('''epoch_'''  ,		''''''	)	)
																		snake_case_       =     None
												else:
																		snake_case_       =     int(training_difference.replace('''step_'''  ,		''''''	)	)
																		num_epochs -= resume_step // len(SCREAMING_SNAKE_CASE__	)
																		# If resuming by step, we also need to know exactly how far into the DataLoader we went
																		snake_case_       =     (num_epochs * len(SCREAMING_SNAKE_CASE__	)) - resume_step

    # Now we train the model
						for epoch in range(SCREAMING_SNAKE_CASE__	):
												# New Code #
												# context manager to track the peak memory usage during the training epoch
												with TorchTracemalloc() as tracemalloc:
																		model.train()
																		if args.with_tracking:
																								snake_case_       =     0
																		for step, batch in enumerate(SCREAMING_SNAKE_CASE__	):
																								# We need to skip steps until we reach the resumed step
																								if args.resume_from_checkpoint and epoch == 0:
																														if resume_step is not None and step < resume_step:
																																				pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device	)
																								snake_case_       =     model(**SCREAMING_SNAKE_CASE__	)
																								snake_case_       =     outputs.loss
																								snake_case_       =     loss / gradient_accumulation_steps
																								# We keep track of the loss at each epoch
																								if args.with_tracking:
																														total_loss += loss.detach().float()
																								accelerator.backward(SCREAMING_SNAKE_CASE__	)
																								if step % gradient_accumulation_steps == 0:
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()
																														# accelerator.print(lr_scheduler.get_lr())

																								overall_step += 1

																								if isinstance(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__	):
																														snake_case_       =     F'''step_{overall_step}'''
																														if overall_step % checkpointing_steps == 0:
																																				if args.output_dir is not None:
																																										snake_case_       =     os.path.join(args.output_dir  ,		SCREAMING_SNAKE_CASE__	)
																																				accelerator.save_state(SCREAMING_SNAKE_CASE__	)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin	)	)	)
												accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used	)	)
												accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked	)	)
												accelerator.print(
												    '''Total Peak Memory consumed during the train (max): {}'''.format(
												        tracemalloc.peaked + bamb(tracemalloc.begin	)	)	)
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin	),
																		    }  ,		step=SCREAMING_SNAKE_CASE__  ,		)

												# New Code #
												# context manager to track the peak memory usage during the evaluation
												with TorchTracemalloc() as tracemalloc:
																		model.eval()
																		for step, batch in enumerate(SCREAMING_SNAKE_CASE__	):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device	)
																								with torch.no_grad():
																														snake_case_       =     model(**SCREAMING_SNAKE_CASE__	)
																								snake_case_       =     outputs.logits.argmax(dim=-1	)
																								snake_case_,   snake_case_       =     accelerator.gather_for_metrics((predictions, batch['''labels'''])	)
																								metric.add_batch(
																								    predictions=SCREAMING_SNAKE_CASE__  ,		references=SCREAMING_SNAKE_CASE__  ,		)

																		snake_case_       =     metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(F'''epoch {epoch}:'''  ,		SCREAMING_SNAKE_CASE__	)
																		if args.with_tracking:
																								accelerator.log(
																								    {
																								        '''accuracy''': eval_metric['''accuracy'''],
																								        '''f1''': eval_metric['''f1'''],
																								        '''train_loss''': total_loss.item() / len(SCREAMING_SNAKE_CASE__	),
																								    }  ,		step=SCREAMING_SNAKE_CASE__  ,		)

																		if checkpointing_steps == "epoch":
																								snake_case_       =     F'''epoch_{epoch}'''
																								if args.output_dir is not None:
																														snake_case_       =     os.path.join(args.output_dir  ,		SCREAMING_SNAKE_CASE__	)
																								accelerator.save_state(SCREAMING_SNAKE_CASE__	)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin	)	)	)
												accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used	)	)
												accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked	)	)
												accelerator.print(
												    '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin	)	)	)
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin	),
																		    }  ,		step=SCREAMING_SNAKE_CASE__  ,		)

						if args.with_tracking:
												accelerator.end_training()


def   __SCREAMING_SNAKE_CASE ():
						snake_case_       =     argparse.ArgumentParser(description='''Simple example of training script.'''	)
						parser.add_argument(
						    '''--mixed_precision'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=SCREAMING_SNAKE_CASE__  ,		choices=['''no''', '''fp16''', '''bf16''', '''fp8''']  ,		help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''  ,		)
						parser.add_argument('''--cpu'''  ,		action='''store_true'''  ,		help='''If passed, will train on the CPU.'''	)
						parser.add_argument(
						    '''--checkpointing_steps'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=SCREAMING_SNAKE_CASE__  ,		help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''  ,		)
						parser.add_argument(
						    '''--resume_from_checkpoint'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=SCREAMING_SNAKE_CASE__  ,		help='''If the training should continue from a checkpoint folder.'''  ,		)
						parser.add_argument(
						    '''--with_tracking'''  ,		action='''store_true'''  ,		help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''  ,		)
						parser.add_argument(
						    '''--output_dir'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default='''.'''  ,		help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''  ,		)
						parser.add_argument(
						    '''--logging_dir'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default='''logs'''  ,		help='''Location on where to store experiment tracking logs`'''  ,		)
						parser.add_argument(
						    '''--model_name_or_path'''  ,		type=SCREAMING_SNAKE_CASE__  ,		help='''Path to pretrained model or model identifier from huggingface.co/models.'''  ,		required=SCREAMING_SNAKE_CASE__  ,		)
						snake_case_       =     parser.parse_args()
						snake_case_       =     {'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
						training_function(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__	)


if __name__ == "__main__":
					main()


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase_	     =     logging.get_logger(__name__)

lowerCAmelCase_	     =     {'''vocab_file''': '''vocab.txt'''}

lowerCAmelCase_	     =     {
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

lowerCAmelCase_	     =     {
    '''tau/splinter-base''': 5_12,
    '''tau/splinter-base-qass''': 5_12,
    '''tau/splinter-large''': 5_12,
    '''tau/splinter-large-qass''': 5_12,
}

lowerCAmelCase_	     =     {
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}





class      snake_case_       (		__A ):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE					:       List[str]						      =    VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE					:       List[str]						      =    PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE					:       Optional[int]						      =    PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE					:       List[str]						      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE					:       Any						      =    SplinterTokenizer







			def __init__(					self			:    List[str]							,       _UpperCamelCase			:    Dict=None							,       _UpperCamelCase			:    Union[str, Any]=None							,       _UpperCamelCase			:    Union[str, Any]=True							,       _UpperCamelCase			:    List[Any]="[UNK]"							,       _UpperCamelCase			:    List[str]="[SEP]"							,       _UpperCamelCase			:    Optional[int]="[PAD]"							,       _UpperCamelCase			:    Tuple="[CLS]"							,       _UpperCamelCase			:    Optional[int]="[MASK]"							,       _UpperCamelCase			:    str="[QUESTION]"							,       _UpperCamelCase			:    Optional[Any]=True							,       _UpperCamelCase			:    Dict=None							,       **_UpperCamelCase			:    Optional[int]							,       )				->Union[str, Any]:
									super().__init__(
									    _UpperCamelCase							,       tokenizer_file=_UpperCamelCase							,       do_lower_case=_UpperCamelCase							,       unk_token=_UpperCamelCase							,       sep_token=_UpperCamelCase							,       pad_token=_UpperCamelCase							,       cls_token=_UpperCamelCase							,       mask_token=_UpperCamelCase							,       tokenize_chinese_chars=_UpperCamelCase							,       strip_accents=_UpperCamelCase							,       additional_special_tokens=(question_token,)							,       **_UpperCamelCase							,       )

									snake_case_       =     json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
									if (
									    pre_tok_state.get('''lowercase'''							,       _UpperCamelCase							) != do_lower_case
									    or pre_tok_state.get('''strip_accents'''							,       _UpperCamelCase							) != strip_accents
									):
															snake_case_       =     getattr(_UpperCamelCase							,       pre_tok_state.pop('''type'''							)							)
															snake_case_       =     do_lower_case
															snake_case_       =     strip_accents
															snake_case_       =     pre_tok_class(**_UpperCamelCase							)

									snake_case_       =     do_lower_case







			@property
			def     snake_case__(					self			:    Optional[int]							)				->int:
									return self.convert_tokens_to_ids(self.question_token							)







			def     snake_case__(					self			:    Dict							,       _UpperCamelCase			:    List[int]							,       _UpperCamelCase			:    Optional[List[int]] = None							)				->List[int]:
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									snake_case_       =     [self.cls_token_id]
									snake_case_       =     [self.sep_token_id]
									snake_case_       =     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''							)]
									if self.padding_side == "right":
															# Input is question-then-context
															return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
															# Input is context-then-question
															return cls + token_ids_a + sep + token_ids_a + question_suffix + sep







			def     snake_case__(					self			:    str							,       _UpperCamelCase			:    List[int]							,       _UpperCamelCase			:    Optional[List[int]] = None							)				->List[int]:
									snake_case_       =     [self.sep_token_id]
									snake_case_       =     [self.cls_token_id]
									snake_case_       =     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''							)]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]

									if self.padding_side == "right":
															# Input is question-then-context
															return len(cls + token_ids_a + question_suffix + sep							) * [0] + len(token_ids_a + sep							) * [1]
									else:
															# Input is context-then-question
															return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + question_suffix + sep							) * [1]







			def     snake_case__(					self			:    Union[str, Any]							,       _UpperCamelCase			:    str							,       _UpperCamelCase			:    Optional[str] = None							)				->Tuple[str]:
									snake_case_       =     self._tokenizer.model.save(_UpperCamelCase							,       name=_UpperCamelCase							)
									return tuple(_UpperCamelCase							)