






import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[str]     ,			__lowercase				:     List[Any]      )  -> Optional[int]:





    """simple docstring"""

    if isinstance(__lowercase     ,			(list, tuple)      ):
        return honor_type(__lowercase     ,			(move_to_device(__lowercase     ,			__lowercase      ) for t in state)      )
    elif isinstance(__lowercase     ,			__lowercase      ):
        return type(__lowercase      )({k: move_to_device(__lowercase     ,			__lowercase      ) for k, v in state.items()}      )
    elif isinstance(__lowercase     ,			torch.Tensor      ):
        return state.to(__lowercase      )
    return state






class  __lowercase (		torch.optim.Optimizer					):







     '''simple docstring'''




     def __init__(			self    :     Union[str, Any]	,		UpperCamelCase_    :     List[str]	,		UpperCamelCase_    :     List[Any]=True	,		UpperCamelCase_    :     Tuple=None			):


         """simple docstring"""



         __A             =		optimizer
         __A             =		scaler
         __A             =		AcceleratorState()
         __A             =		GradientState()
         __A             =		device_placement
         __A             =		False
         __A             =		None

         # Handle device placement
         if device_placement:
             __A             =		self.optimizer.state_dict()
             if self.accelerator_state.distributed_type == DistributedType.TPU:
                 xm.send_cpu_data_to_device(UpperCamelCase_	,		self.accelerator_state.device			)
             else:
                 __A             =		move_to_device(UpperCamelCase_	,		self.accelerator_state.device			)
             self.optimizer.load_state_dict(UpperCamelCase_			)




     @property
     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         return self.optimizer.state




     @state.setter
     def 						lowerCAmelCase_       (			self    :     Any	,		UpperCamelCase_    :     Union[str, Any]			):


         """simple docstring"""



         __A             =		state




     @property
     def 						lowerCAmelCase_       (			self    :     List[Any]			):


         """simple docstring"""



         return self.optimizer.param_groups




     @param_groups.setter
     def 						lowerCAmelCase_       (			self    :     List[Any]	,		UpperCamelCase_    :     Any			):


         """simple docstring"""



         __A             =		param_groups




     @property
     def 						lowerCAmelCase_       (			self    :     int			):


         """simple docstring"""



         return self.optimizer.defaults




     @defaults.setter
     def 						lowerCAmelCase_       (			self    :     List[Any]	,		UpperCamelCase_    :     Union[str, Any]			):


         """simple docstring"""



         __A             =		defaults




     def 						lowerCAmelCase_       (			self    :     str	,		UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         self.optimizer.add_param_group(UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Dict	,		UpperCamelCase_    :     Dict			):


         """simple docstring"""



         if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
             xm.send_cpu_data_to_device(UpperCamelCase_	,		self.accelerator_state.device			)
         self.optimizer.load_state_dict(UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         return self.optimizer.state_dict()




     def 						lowerCAmelCase_       (			self    :     Dict	,		UpperCamelCase_    :     List[Any]=None			):


         """simple docstring"""



         if self.gradient_state.sync_gradients:
             __A             =		"""set_to_none""" in inspect.signature(self.optimizer.zero_grad			).parameters
             if accept_arg:
                 if set_to_none is None:
                     __A             =		False
                 self.optimizer.zero_grad(set_to_none=UpperCamelCase_			)
             else:
                 if set_to_none is not None:
                     raise ValueError("""`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer."""			)
                 self.optimizer.zero_grad()




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     List[Any]=None			):


         """simple docstring"""



         if self.gradient_state.sync_gradients:
             if self.accelerator_state.distributed_type == DistributedType.TPU:
                 __A             =		{"""closure""": closure} if closure is not None else {}
                 xm.optimizer_step(self.optimizer	,		optimizer_args=UpperCamelCase_			)
             elif self.scaler is not None:
                 __A             =		False
                 if self._last_scale is None:
                     # `get_scale` is an async operation requiring full synchronization
                     # on CPU and GPUs before finishing. As a result, we store away
                     # the prior one to reduce the call overhead
                     __A             =		self.scaler.get_scale()
                     __A             =		True
                 self.scaler.step(self.optimizer	,		UpperCamelCase_			)
                 self.scaler.update()
                 __A             =		self.scaler.get_scale()
                 if not new_scale:
                     # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
                     __A             =		scale_after < self._last_scale
                 __A             =		scale_after

             else:
                 self.optimizer.step(UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     Any			):


         """simple docstring"""



         for param_group in self.optimizer.param_groups:
             __A             =		[parameters_map.get(UpperCamelCase_	,		UpperCamelCase_			) for p in param_group["""params"""]]




     @property
     def 						lowerCAmelCase_       (			self    :     Any			):


         """simple docstring"""



         warnings.warn(
             """The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use """
             """`optimizer.step_was_skipped` instead."""	,		UpperCamelCase_	,		)
         return self._is_overflow




     @property
     def 						lowerCAmelCase_       (			self    :     int			):


         """simple docstring"""



         return self._is_overflow




     def __getstate__(			self    :     Dict			):


         """simple docstring"""



         return self.__dict__.copy()





     def __setstate__(			self    :     str	,		UpperCamelCase_    :     str			):


         """simple docstring"""



         self.__dict__.update(UpperCamelCase_			)












import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__a :    Optional[Any]								=				get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class  __lowercase (		lowercase_     , unittest.TestCase					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			ErnieMTokenizer
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			True
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			False




     def 						lowerCAmelCase_       (			self    :     Optional[Any]			):


         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)
         tokenizer.save_pretrained(self.tmpdirname			)




     def 						lowerCAmelCase_       (			self    :     Optional[Any]	,		UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         __A             =		"""this is a test"""
         __A             =		"""this is a test"""
         return input_text, output_text




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		"""<pad>"""
         __A             =		0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_			)	,		UpperCamelCase_			)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_			)	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         __A             =		list(self.get_tokenizer().get_vocab().keys()			)

         self.assertEqual(vocab_keys[0]	,		"""<pad>"""			)
         self.assertEqual(vocab_keys[1]	,		"""<unk>"""			)
         self.assertEqual(vocab_keys[-1]	,		"""▁eloquent"""			)
         self.assertEqual(len(UpperCamelCase_			)	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         if not self.test_rust_tokenizer:
             return

         __A             =		self.get_tokenizer()
         __A             =		self.get_rust_tokenizer()

         __A             =		"""I was born in 92000, and this is falsé."""

         __A             =		tokenizer.tokenize(UpperCamelCase_			)
         __A             =		rust_tokenizer.tokenize(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		self.get_rust_tokenizer()
         __A             =		tokenizer.encode(UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		do_lower_case=UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.tokenize("""This is a test"""			)
         self.assertListEqual(UpperCamelCase_	,		["""▁this""", """▁is""", """▁a""", """▁test"""]			)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_			)	,		[48, 25, 21, 1_289]			)

         __A             =		tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
         # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]			)
         __A             =		tokenizer.convert_tokens_to_ids(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		[31, 23, 386, 19, 518, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]			)

         __A             =		tokenizer.convert_ids_to_tokens(UpperCamelCase_			)
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]	,		)




     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.encode("""sequence builders"""			)
         __A             =		tokenizer.encode("""multi-sequence build"""			)

         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			)
         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_	,		UpperCamelCase_			)

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
             tokenizer.sep_token_id
         ] + text_a + [tokenizer.sep_token_id]





     @slow
     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		{"""input_ids""": [[0, 11_062, 82_772, 7, 15, 82_772, 538, 51_529, 237, 17_198, 1_290, 206, 9, 215_175, 1_314, 136, 17_198, 1_290, 206, 9, 56_359, 42, 122_009, 9, 16_466, 16, 87_344, 4_537, 9, 4_717, 78_381, 6, 159_958, 7, 15, 24_480, 618, 4, 527, 22_693, 9, 304, 4, 2_777, 24_480, 9_874, 4, 43_523, 594, 4, 803, 18_392, 33_189, 18, 4, 43_523, 24_447, 5, 5, 5, 16, 100, 24_955, 83_658, 9_626, 144_057, 15, 839, 22_335, 16, 136, 24_955, 83_658, 83_479, 15, 39_102, 724, 16, 678, 645, 6_460, 1_328, 4_589, 42, 122_009, 115_774, 23, 3_559, 1_328, 46_876, 7, 136, 53_894, 1_940, 42_227, 41_159, 17_721, 823, 425, 4, 27_512, 98_722, 206, 136, 5_531, 4_970, 919, 17_336, 5, 2], [0, 20_080, 618, 83, 82_775, 47, 479, 9, 1_517, 73, 53_894, 333, 80_581, 110_117, 18_811, 5_256, 1_295, 51, 152_526, 297, 7_986, 390, 124_416, 538, 35_431, 214, 98, 15_044, 25_737, 136, 7_108, 43_701, 23, 756, 135_355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63_773, 119_455, 6, 147_797, 88_203, 7, 645, 70, 21, 3_285, 10_269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=UpperCamelCase_	,		model_name="""susnato/ernie-m-base_pytorch"""	,		sequences=[
                 """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
                 """general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
                 """Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained """
                 """models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
                 """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
                 """conditioning on both left and right context in all layers.""",
                 """The quick brown fox jumps over the lazy dog.""",
             ]	,		)




