



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 						_snake_case (     lowerCamelCase   ,unittest.TestCase			):





	"""simple docstring"""



	lowerCamelCase_       =     RealmTokenizer
	lowerCamelCase_       =     RealmTokenizerFast
	lowerCamelCase_       =     True
	lowerCamelCase_       =     True
	lowerCamelCase_       =     filter_non_english
	def lowercase_						(  self   )		->							Any:




			"""simple docstring"""







			super().setUp()

			_A          =      [
			    '''[UNK]''',
			    '''[CLS]''',
			    '''[SEP]''',
			    '''[PAD]''',
			    '''[MASK]''',
			    '''want''',
			    '''##want''',
			    '''##ed''',
			    '''wa''',
			    '''un''',
			    '''runn''',
			    '''##ing''',
			    ''',''',
			    '''low''',
			    '''lowest''',
			]
			_A          =      os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES['''vocab_file''']   )
			with open(self.vocab_file		,      '''w'''		,      encoding='''utf-8'''   ) as vocab_writer:
					vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
	def lowercase_						(  self		,      a   )		->							Union[str, Any]:




			"""simple docstring"""







			_A          =      '''UNwant\u00E9d,running'''
			_A          =      '''unwanted, running'''
			return input_text, output_text
	def lowercase_						(  self   )		->							Optional[int]:




			"""simple docstring"""







			_A          =      self.tokenizer_class(self.vocab_file   )

			_A          =      tokenizer.tokenize('''UNwant\u00E9d,running'''   )
			self.assertListEqual(a		,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
			self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )		,      [9, 6, 7, 1_2, 1_0, 1_1]   )
	def lowercase_						(  self   )		->							Union[str, Any]:




			"""simple docstring"""







			if not self.test_rust_tokenizer:
					return

			_A          =      self.get_tokenizer()
			_A          =      self.get_rust_tokenizer()

			_A          =      '''UNwant\u00E9d,running'''

			_A          =      tokenizer.tokenize(a   )
			_A          =      rust_tokenizer.tokenize(a   )
			self.assertListEqual(a		,      a   )

			_A          =      tokenizer.encode(a		,      add_special_tokens=a   )
			_A          =      rust_tokenizer.encode(a		,      add_special_tokens=a   )
			self.assertListEqual(a		,      a   )

			_A          =      self.get_rust_tokenizer()
			_A          =      tokenizer.encode(a   )
			_A          =      rust_tokenizer.encode(a   )
			self.assertListEqual(a		,      a   )

			# With lower casing
			_A          =      self.get_tokenizer(do_lower_case=a   )
			_A          =      self.get_rust_tokenizer(do_lower_case=a   )

			_A          =      '''UNwant\u00E9d,running'''

			_A          =      tokenizer.tokenize(a   )
			_A          =      rust_tokenizer.tokenize(a   )
			self.assertListEqual(a		,      a   )

			_A          =      tokenizer.encode(a		,      add_special_tokens=a   )
			_A          =      rust_tokenizer.encode(a		,      add_special_tokens=a   )
			self.assertListEqual(a		,      a   )

			_A          =      self.get_rust_tokenizer()
			_A          =      tokenizer.encode(a   )
			_A          =      rust_tokenizer.encode(a   )
			self.assertListEqual(a		,      a   )
	def lowercase_						(  self   )		->							List[Any]:




			"""simple docstring"""







			_A          =      BasicTokenizer()

			self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )		,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )
	def lowercase_						(  self   )		->							List[Any]:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
			self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
	def lowercase_						(  self   )		->							Dict:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
			self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''h\u00E9llo''']   )
	def lowercase_						(  self   )		->							Optional[Any]:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
			self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
	def lowercase_						(  self   )		->							Optional[Any]:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
			self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
	def lowercase_						(  self   )		->							List[Any]:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
	def lowercase_						(  self   )		->							int:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
	def lowercase_						(  self   )		->							List[str]:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
	def lowercase_						(  self   )		->							Tuple:




			"""simple docstring"""







			_A          =      BasicTokenizer(do_lower_case=a		,      never_split=['''[UNK]''']   )

			self.assertListEqual(
			    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )
	def lowercase_						(  self   )		->							Tuple:




			"""simple docstring"""







			_A          =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

			_A          =      {}
			for i, token in enumerate(a   ):
					_A          =      i
			_A          =      WordpieceTokenizer(vocab=a		,      unk_token='''[UNK]'''   )

			self.assertListEqual(tokenizer.tokenize(''''''   )		,      []   )

			self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )		,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

			self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )		,      ['''[UNK]''', '''runn''', '''##ing''']   )
	def lowercase_						(  self   )		->							Tuple:




			"""simple docstring"""







			self.assertTrue(_is_whitespace(''' '''   )   )
			self.assertTrue(_is_whitespace('''\t'''   )   )
			self.assertTrue(_is_whitespace('''\r'''   )   )
			self.assertTrue(_is_whitespace('''\n'''   )   )
			self.assertTrue(_is_whitespace('''\u00A0'''   )   )

			self.assertFalse(_is_whitespace('''A'''   )   )
			self.assertFalse(_is_whitespace('''-'''   )   )
	def lowercase_						(  self   )		->							Optional[int]:




			"""simple docstring"""







			self.assertTrue(_is_control('''\u0005'''   )   )

			self.assertFalse(_is_control('''A'''   )   )
			self.assertFalse(_is_control(''' '''   )   )
			self.assertFalse(_is_control('''\t'''   )   )
			self.assertFalse(_is_control('''\r'''   )   )
	def lowercase_						(  self   )		->							int:




			"""simple docstring"""







			self.assertTrue(_is_punctuation('''-'''   )   )
			self.assertTrue(_is_punctuation('''$'''   )   )
			self.assertTrue(_is_punctuation('''`'''   )   )
			self.assertTrue(_is_punctuation('''.'''   )   )

			self.assertFalse(_is_punctuation('''A'''   )   )
			self.assertFalse(_is_punctuation(''' '''   )   )
	def lowercase_						(  self   )		->							Union[str, Any]:




			"""simple docstring"""







			_A          =      self.get_tokenizer()

			# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
			self.assertListEqual([tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]   )

			if self.test_rust_tokenizer:
					_A          =      self.get_rust_tokenizer()
					self.assertListEqual(
					    [rust_tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]   )
	@slow
	def lowercase_						(  self   )		->							Dict:




			"""simple docstring"""







			_A          =      self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

			_A          =      tokenizer.encode('''sequence builders'''		,      add_special_tokens=a   )
			_A          =      tokenizer.encode('''multi-sequence build'''		,      add_special_tokens=a   )

			_A          =      tokenizer.build_inputs_with_special_tokens(a   )
			_A          =      tokenizer.build_inputs_with_special_tokens(a		,      a   )

			assert encoded_sentence == [1_0_1] + text + [1_0_2]
			assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
	def lowercase_						(  self   )		->							str:




			"""simple docstring"""







			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
							_A          =      self.rust_tokenizer_class.from_pretrained(a		,      **a   )

							_A          =      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
							_A          =      tokenizer_r.encode_plus(
							    a		,      return_attention_mask=a		,      return_token_type_ids=a		,      return_offsets_mapping=a		,      add_special_tokens=a		,      )

							_A          =      tokenizer_r.do_lower_case if hasattr(a		,      '''do_lower_case'''   ) else False
							_A          =      (
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''A'''),
							        ((1, 2), ''','''),
							        ((3, 5), '''na'''),
							        ((5, 6), '''##ï'''),
							        ((6, 8), '''##ve'''),
							        ((9, 1_5), tokenizer_r.mask_token),
							        ((1_6, 2_1), '''Allen'''),
							        ((2_1, 2_3), '''##NL'''),
							        ((2_3, 2_4), '''##P'''),
							        ((2_5, 3_3), '''sentence'''),
							        ((3_3, 3_4), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''a'''),
							        ((1, 2), ''','''),
							        ((3, 8), '''naive'''),
							        ((9, 1_5), tokenizer_r.mask_token),
							        ((1_6, 2_1), '''allen'''),
							        ((2_1, 2_3), '''##nl'''),
							        ((2_3, 2_4), '''##p'''),
							        ((2_5, 3_3), '''sentence'''),
							        ((3_3, 3_4), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results]		,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
							self.assertEqual([e[0] for e in expected_results]		,      tokens['''offset_mapping''']   )



	@slow
	def lowercase_						(  self   )		->							Optional[Any]:




			"""simple docstring"""







			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
							_A          =      self.rust_tokenizer_class.from_pretrained(a		,      **a   )
							_A          =      self.tokenizer_class.from_pretrained(a		,      **a   )
							_A          =      [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

							_A          =      tokenizer_r.batch_encode_candidates(a		,      max_length=1_0		,      return_tensors='''np'''   )
							_A          =      tokenizer_p.batch_encode_candidates(a		,      max_length=1_0		,      return_tensors='''np'''   )

							_A          =      (2, 2, 1_0)

							self.assertEqual(encoded_sentence_r['''input_ids'''].shape		,      a   )
							self.assertEqual(encoded_sentence_r['''attention_mask'''].shape		,      a   )
							self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape		,      a   )

							self.assertEqual(encoded_sentence_p['''input_ids'''].shape		,      a   )
							self.assertEqual(encoded_sentence_p['''attention_mask'''].shape		,      a   )
							self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape		,      a   )




import warnings

from .generation import FlaxGenerationMixin



class 						_snake_case (     lowerCamelCase			):





 """simple docstring"""



 warnings.warn(
     '''Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will '''
     '''be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead.'''   ,lowerCamelCase   ,)