
'''simple docstring'''



import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A       =      get_tests_dir("fixtures/spiece.model")





@require_sentencepiece
@require_tokenizers
class      A							(    __UpperCAmelCase						,  unittest.TestCase			):
			lowerCamelCase							:							List[str]    						=					ErnieMTokenizer
			lowerCamelCase							:							Optional[int]    						=					False
			lowerCamelCase							:							Any    						=					True
			lowerCamelCase							:							List[Any]    						=					False
			lowerCamelCase							:							Any    						=					False



			def A__	(   self    )     ->						Any:




									'''simple docstring'''



									super().setUp()

									# We have a SentencePiece fixture for testing
									lowercase__								=    ErnieMTokenizer(lowerCamelCase__    ,       unk_token="""<unk>"""    ,       pad_token="""<pad>"""    )
									tokenizer.save_pretrained(self.tmpdirname    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    """this is a test"""
									lowercase__								=    """this is a test"""
									return input_text, output_text



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									lowercase__								=    """<pad>"""
									lowercase__								=    0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__    )    ,       lowerCamelCase__    )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__    )    ,       lowerCamelCase__    )



			def A__	(   self    )     ->						str:




									'''simple docstring'''



									lowercase__								=    list(self.get_tokenizer().get_vocab().keys()    )

									self.assertEqual(vocab_keys[0]    ,       """<pad>"""    )
									self.assertEqual(vocab_keys[1]    ,       """<unk>"""    )
									self.assertEqual(vocab_keys[-1]    ,       """▁eloquent"""    )
									self.assertEqual(len(lowerCamelCase__    )    ,       30_000    )



			def A__	(   self    )     ->						int:




									'''simple docstring'''



									self.assertEqual(self.get_tokenizer().vocab_size    ,       30_000    )



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									if not self.test_rust_tokenizer:
															return

									lowercase__								=    self.get_tokenizer()
									lowercase__								=    self.get_rust_tokenizer()

									lowercase__								=    """I was born in 92000, and this is falsé."""

									lowercase__								=    tokenizer.tokenize(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    self.get_rust_tokenizer()
									lowercase__								=    tokenizer.encode(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )



			def A__	(   self    )     ->						int:




									'''simple docstring'''



									lowercase__								=    ErnieMTokenizer(lowerCamelCase__    ,       do_lower_case=lowerCamelCase__    ,       unk_token="""<unk>"""    ,       pad_token="""<pad>"""    )

									lowercase__								=    tokenizer.tokenize("""This is a test"""    )
									self.assertListEqual(lowerCamelCase__    ,       ["""▁this""", """▁is""", """▁a""", """▁test"""]    )

									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )    ,       [48, 25, 21, 1_289]    )

									lowercase__								=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
									# ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
									self.assertListEqual(
									    lowerCamelCase__    ,       ["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]    )
									lowercase__								=    tokenizer.convert_tokens_to_ids(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       [31, 23, 386, 19, 518, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]    )

									lowercase__								=    tokenizer.convert_ids_to_tokens(lowerCamelCase__    )
									self.assertListEqual(
									    lowerCamelCase__    ,       ["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]    ,       )



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									lowercase__								=    ErnieMTokenizer(lowerCamelCase__    ,       unk_token="""<unk>"""    ,       pad_token="""<pad>"""    )

									lowercase__								=    tokenizer.encode("""sequence builders"""    )
									lowercase__								=    tokenizer.encode("""multi-sequence build"""    )

									lowercase__								=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    )
									lowercase__								=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    ,       lowerCamelCase__    )

									assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
									assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
									    tokenizer.sep_token_id
									] + text_a + [tokenizer.sep_token_id]



			@slow
			def A__	(   self    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    {"""input_ids""": [[0, 11_062, 82_772, 7, 15, 82_772, 538, 51_529, 237, 17_198, 1_290, 206, 9, 215_175, 1_314, 136, 17_198, 1_290, 206, 9, 56_359, 42, 122_009, 9, 16_466, 16, 87_344, 4_537, 9, 4_717, 78_381, 6, 159_958, 7, 15, 24_480, 618, 4, 527, 22_693, 9, 304, 4, 2_777, 24_480, 9_874, 4, 43_523, 594, 4, 803, 18_392, 33_189, 18, 4, 43_523, 24_447, 5, 5, 5, 16, 100, 24_955, 83_658, 9_626, 144_057, 15, 839, 22_335, 16, 136, 24_955, 83_658, 83_479, 15, 39_102, 724, 16, 678, 645, 6_460, 1_328, 4_589, 42, 122_009, 115_774, 23, 3_559, 1_328, 46_876, 7, 136, 53_894, 1_940, 42_227, 41_159, 17_721, 823, 425, 4, 27_512, 98_722, 206, 136, 5_531, 4_970, 919, 17_336, 5, 2], [0, 20_080, 618, 83, 82_775, 47, 479, 9, 1_517, 73, 53_894, 333, 80_581, 110_117, 18_811, 5_256, 1_295, 51, 152_526, 297, 7_986, 390, 124_416, 538, 35_431, 214, 98, 15_044, 25_737, 136, 7_108, 43_701, 23, 756, 135_355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63_773, 119_455, 6, 147_797, 88_203, 7, 645, 70, 21, 3_285, 10_269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=lowerCamelCase__    ,       model_name="""susnato/ernie-m-base_pytorch"""    ,       sequences=[
									        """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
									        """general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
									        """Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained """
									        """models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
									        """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
									        """conditioning on both left and right context in all layers.""",
									        """The quick brown fox jumps over the lazy dog.""",
									    ]    ,       )








'''simple docstring'''



def 	_A						(							lowercase__		):
						return " ".join(sorted(set(sentence.split()		)		)		)


if __name__ == "__main__":
				import doctest

				doctest.testmod()






