




import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class 		UpperCAmelCase       (    tf.keras.layers.Layer      ):







						'''simple docstring'''





						def __init__(						self							:					Union[str, Any]	,     __lowercase							:					List	,     __lowercase							:					bool	,     __lowercase							:					int = None	,     __lowercase							:					int = None	,     __lowercase							:					int = None	,     __lowercase							:					str = "longest"	,     __lowercase							:					bool = True	,     __lowercase							:					int = 5_12	,     __lowercase							:					int = None	,     __lowercase							:					bool = True	,     __lowercase							:					bool = True	,     __lowercase							:					bool = True	,     **__lowercase							:					Tuple	,     ):






							"""simple docstring"""






							super().__init__()
							if use_fast_bert_tokenizer:
								snake_case_		   =					FastBertTokenizer(
								    __lowercase	,     token_out_type=tf.intaa	,     lower_case_nfd_strip_accents=__lowercase	,     **__lowercase						)
							else:
								snake_case_		   =					tf.lookup.StaticVocabularyTable(
								    tf.lookup.KeyValueTensorInitializer(
								        keys=__lowercase	,     key_dtype=tf.string	,     values=tf.range(tf.size(__lowercase	,     out_type=tf.intaa						)	,     dtype=tf.intaa						)	,     value_dtype=tf.intaa	,     )	,     num_oov_buckets=1	,     )
								snake_case_		   =					BertTokenizerLayer(
								    __lowercase	,     token_out_type=tf.intaa	,     lower_case=__lowercase	,     **__lowercase						)

							snake_case_		   =					vocab_list
							snake_case_		   =					do_lower_case
							snake_case_		   =					cls_token_id or vocab_list.index("[CLS]"						)
							snake_case_		   =					sep_token_id or vocab_list.index("[SEP]"						)
							snake_case_		   =					pad_token_id or vocab_list.index("[PAD]"						)
							snake_case_		   =					ShrinkLongestTrimmer(max_length - 3	,     axis=1						)  # Allow room for special tokens
							snake_case_		   =					max_length
							snake_case_		   =					padding
							snake_case_		   =					truncation
							snake_case_		   =					pad_to_multiple_of
							snake_case_		   =					return_token_type_ids
							snake_case_		   =					return_attention_mask






						@classmethod
						def 	snake_case__						(						cls							:					Any	,     __lowercase							:					"PreTrainedTokenizerBase"	,     **__lowercase							:					Any						):  # noqa: F821






							"""simple docstring"""






							snake_case_		   =					kwargs.pop("do_lower_case"	,     __lowercase						)
							snake_case_		   =					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
							snake_case_		   =					kwargs.pop("cls_token_id"	,     __lowercase						)
							snake_case_		   =					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
							snake_case_		   =					kwargs.pop("sep_token_id"	,     __lowercase						)
							snake_case_		   =					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
							snake_case_		   =					kwargs.pop("pad_token_id"	,     __lowercase						)
							snake_case_		   =					tokenizer.pad_token_id if pad_token_id is None else pad_token_id

							snake_case_		   =					tokenizer.get_vocab()
							snake_case_		   =					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]	,     key=lambda __lowercase						: x[1]						)
							snake_case_		   =					[entry[0] for entry in vocab]
							return cls(
							    vocab_list=__lowercase	,     do_lower_case=__lowercase	,     cls_token_id=__lowercase	,     sep_token_id=__lowercase	,     pad_token_id=__lowercase	,     **__lowercase	,     )






						@classmethod
						def 	snake_case__						(						cls							:					str	,     __lowercase							:					Union[str, os.PathLike]	,     *__lowercase							:					Optional[int]	,     **__lowercase							:					int						):






							"""simple docstring"""






							try:
								snake_case_		   =					BertTokenizer.from_pretrained(__lowercase	,     *__lowercase	,     **__lowercase						)
							except:  # noqa: E722
								from .tokenization_bert_fast import BertTokenizerFast

								snake_case_		   =					BertTokenizerFast.from_pretrained(__lowercase	,     *__lowercase	,     **__lowercase						)
							return cls.from_tokenizer(__lowercase	,     **__lowercase						)






						def 	snake_case__						(						self							:					int	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							if self.do_lower_case:
								snake_case_		   =					case_fold_utfa(__lowercase						)
							snake_case_		   =					self.tf_tokenizer.tokenize(__lowercase						)
							return tokens.merge_dims(1	,     -1						)






						def 	snake_case__						(						self							:					Dict	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Optional[int]=None	,     __lowercase							:					Any=None	,     __lowercase							:					str=None	,     __lowercase							:					Optional[int]=None	,     __lowercase							:					List[str]=None	,     __lowercase							:					str=None	,     __lowercase							:					Tuple=None	,     ):






							"""simple docstring"""






							if padding is None:
								snake_case_		   =					self.padding
							if padding not in ("longest", "max_length"):
								raise ValueError("Padding must be either 'longest' or 'max_length'!"						)
							if max_length is not None and text_pair is not None:
								# Because we have to instantiate a Trimmer to do it properly
								raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"						)
							if max_length is None:
								snake_case_		   =					self.max_length
							if truncation is None:
								snake_case_		   =					self.truncation
							if pad_to_multiple_of is None:
								snake_case_		   =					self.pad_to_multiple_of
							if return_token_type_ids is None:
								snake_case_		   =					self.return_token_type_ids
							if return_attention_mask is None:
								snake_case_		   =					self.return_attention_mask
							if not isinstance(__lowercase	,     tf.Tensor						):
								snake_case_		   =					tf.convert_to_tensor(__lowercase						)
							if text_pair is not None and not isinstance(__lowercase	,     tf.Tensor						):
								snake_case_		   =					tf.convert_to_tensor(__lowercase						)
							if text_pair is not None:
								if text.shape.rank > 1:
									raise ValueError("text argument should not be multidimensional when a text pair is supplied!"						)
								if text_pair.shape.rank > 1:
									raise ValueError("text_pair should not be multidimensional!"						)
							if text.shape.rank == 2:
								snake_case_   ,				snake_case_		   =					text[:, 0], text[:, 1]
							snake_case_		   =					self.unpaired_tokenize(__lowercase						)
							if text_pair is None:  # Unpaired text
								if truncation:
									snake_case_		   =					text[:, : max_length - 2]  # Allow room for special tokens
								snake_case_   ,				snake_case_		   =					combine_segments(
								    (text,)	,     start_of_sequence_id=self.cls_token_id	,     end_of_segment_id=self.sep_token_id						)
							else:  # Paired text
								snake_case_		   =					self.unpaired_tokenize(__lowercase						)
								if truncation:
									snake_case_   ,				snake_case_		   =					self.paired_trimmer.trim([text, text_pair]						)
								snake_case_   ,				snake_case_		   =					combine_segments(
								    (text, text_pair)	,     start_of_sequence_id=self.cls_token_id	,     end_of_segment_id=self.sep_token_id						)
							if padding == "longest":
								snake_case_		   =					input_ids.bounding_shape(axis=1						)
								if pad_to_multiple_of is not None:
									# No ceiling division in tensorflow, so we negate floordiv instead
									snake_case_		   =					pad_to_multiple_of * (-tf.math.floordiv(-pad_length	,     __lowercase						))
							else:
								snake_case_		   =					max_length

							snake_case_   ,				snake_case_		   =					pad_model_inputs(__lowercase	,     max_seq_length=__lowercase	,     pad_value=self.pad_token_id						)
							snake_case_		   =					{"input_ids": input_ids}
							if return_attention_mask:
								snake_case_		   =					attention_mask
							if return_token_type_ids:
								snake_case_   ,				snake_case_		   =					pad_model_inputs(
								    __lowercase	,     max_seq_length=__lowercase	,     pad_value=self.pad_token_id						)
								snake_case_		   =					token_type_ids
							return output






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							return {
							    "vocab_list": self.vocab_list,
							    "do_lower_case": self.do_lower_case,
							    "cls_token_id": self.cls_token_id,
							    "sep_token_id": self.sep_token_id,
							    "pad_token_id": self.pad_token_id,
							}






import unittest
from functools import partial

import torch

from accelerate import Accelerator, debug_launcher
from accelerate.state import AcceleratorState, GradientState
from accelerate.test_utils import require_cpu, require_huggingface_suite
from accelerate.utils import GradientAccumulationPlugin


def 						lowerCamelCase__   (		_A=2					,     _A=True					,     _A=False							):







	'''simple docstring'''





	snake_case_		   =					Accelerator(step_scheduler_with_optimizer=_A					,     split_batches=_A							)
	snake_case_		   =					torch.nn.Linear(2					,     4							)
	snake_case_		   =					torch.optim.AdamW(model.parameters()					,     lr=1.0							)
	snake_case_		   =					torch.optim.lr_scheduler.OneCycleLR(_A					,     max_lr=0.01					,     steps_per_epoch=2					,     epochs=1							)
	snake_case_   ,				snake_case_   ,				snake_case_		   =					accelerator.prepare(_A					,     _A					,     _A							)

	# Optimizer has stepped
	scheduler.step()
	if step_scheduler_with_optimizer or (num_processes == 1):
		assert (
		    scheduler.scheduler.last_epoch == num_processes
		), f"Last Epoch ({scheduler.scheduler.last_epoch}) != Num Processes ({num_processes})"
	else:
		assert (
		    scheduler.scheduler.last_epoch != num_processes
		), f"Last Epoch ({scheduler.scheduler.last_epoch}) == Num Processes ({num_processes})"


def 						lowerCamelCase__   (		_A=2					,     _A=True					,     _A=False							):







	'''simple docstring'''





	snake_case_		   =					Accelerator(step_scheduler_with_optimizer=_A					,     split_batches=_A							)
	snake_case_		   =					torch.nn.Linear(2					,     4							)
	snake_case_		   =					torch.optim.AdamW(model.parameters()					,     lr=1.0							)
	snake_case_		   =					torch.optim.lr_scheduler.LambdaLR(_A					,     lr_lambda=lambda _A							: 1 - n / 10							)
	snake_case_   ,				snake_case_   ,				snake_case_		   =					accelerator.prepare(_A					,     _A					,     _A							)

	# Optimizer has stepped
	snake_case_		   =					False
	scheduler.step()
	snake_case_		   =					1 - (num_processes if (step_scheduler_with_optimizer and not split_batches) else 1) / 10
	assert (
	    scheduler.get_last_lr()[0] == expected_lr
	), f"Wrong lr found at first step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}"

	# Optimizer has not stepped
	snake_case_		   =					True
	scheduler.step()
	if not step_scheduler_with_optimizer:
		snake_case_		   =					1 - 2 / 10
	assert (
	    scheduler.get_last_lr()[0] == expected_lr
	), f"Wrong lr found at second step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}"


def 						lowerCamelCase__   (		_A = 2							):







	'''simple docstring'''





	from transformers import get_linear_schedule_with_warmup

	snake_case_		   =					[1, 2, 4]
	for num_steps in steps:
		snake_case_		   =					GradientAccumulationPlugin(num_steps=_A					,     adjust_scheduler=num_steps > 1							)
		snake_case_		   =					Accelerator(gradient_accumulation_plugin=_A							)
		snake_case_		   =					torch.nn.Linear(2					,     4							)
		snake_case_		   =					torch.optim.AdamW(model.parameters()					,     lr=10.0							)
		snake_case_		   =					get_linear_schedule_with_warmup(optimizer=_A					,     num_warmup_steps=0					,     num_training_steps=20							)

		snake_case_   ,				snake_case_   ,				snake_case_		   =					accelerator.prepare(_A					,     _A					,     _A							)

		for i in range(10 * num_steps							):
			with accelerator.accumulate(_A							):
				optimizer.step()
				scheduler.step()

			if i == (10 * num_steps - 2):
				assert (
				    scheduler.get_last_lr()[0] != 0
				), f"Wrong lr found at second-to-last step, expected non-zero, got {scheduler.get_last_lr()[0]}. num_steps: {num_steps}"
		assert (
		    scheduler.get_last_lr()[0] == 0
		), f"Wrong lr found at last step, expected 0, got {scheduler.get_last_lr()[0]}"
		GradientState._reset_state()






@require_cpu
class 		UpperCAmelCase       (    unittest.TestCase      ):







						'''simple docstring'''





						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							debug_launcher(partial(__lowercase	,     num_processes=1						)	,     num_processes=1						)
							debug_launcher(partial(__lowercase	,     num_processes=1	,     split_batches=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							debug_launcher(partial(__lowercase	,     num_processes=1						)	,     num_processes=1						)
							debug_launcher(partial(__lowercase	,     num_processes=1	,     split_batches=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							debug_launcher(partial(__lowercase	,     num_processes=1	,     step_scheduler_with_optimizer=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							debug_launcher(partial(__lowercase	,     num_processes=1	,     step_scheduler_with_optimizer=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							AcceleratorState._reset_state(__lowercase						)
							debug_launcher(__lowercase						)
							debug_launcher(partial(__lowercase	,     num_processes=1	,     split_batches=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							AcceleratorState._reset_state(__lowercase						)
							debug_launcher(__lowercase						)
							debug_launcher(partial(__lowercase	,     num_processes=1	,     split_batches=__lowercase						)	,     num_processes=1						)






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							AcceleratorState._reset_state(__lowercase						)
							debug_launcher(partial(__lowercase	,     step_scheduler_with_optimizer=__lowercase						)						)






						def 	snake_case__						(						self							:					Optional[Any]						):






							"""simple docstring"""






							AcceleratorState._reset_state(__lowercase						)
							debug_launcher(partial(__lowercase	,     step_scheduler_with_optimizer=__lowercase						)						)






						@require_huggingface_suite
						def 	snake_case__						(						self							:					Optional[Any]						):






							"""simple docstring"""






							AcceleratorState._reset_state(__lowercase						)
							debug_launcher(partial(__lowercase	,     num_processes=1						)						)
							debug_launcher(__lowercase						)
