



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 						_snake_case (     lowerCamelCase   ,unittest.TestCase			):





 """simple docstring"""



 lowerCamelCase_       =     RealmTokenizer
 lowerCamelCase_       =     RealmTokenizerFast
 lowerCamelCase_       =     True
 lowerCamelCase_       =     True
 lowerCamelCase_       =     filter_non_english
 def lowercase_						(  self   )		->							Any:




   """simple docstring"""







   super().setUp()

   _A          =      [
       '''[UNK]''',
       '''[CLS]''',
       '''[SEP]''',
       '''[PAD]''',
       '''[MASK]''',
       '''want''',
       '''##want''',
       '''##ed''',
       '''wa''',
       '''un''',
       '''runn''',
       '''##ing''',
       ''',''',
       '''low''',
       '''lowest''',
   ]
   _A          =      os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES['''vocab_file''']   )
   with open(self.vocab_file		,      '''w'''		,      encoding='''utf-8'''   ) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
 def lowercase_						(  self		,      a   )		->							Union[str, Any]:




   """simple docstring"""







   _A          =      '''UNwant\u00E9d,running'''
   _A          =      '''unwanted, running'''
   return input_text, output_text
 def lowercase_						(  self   )		->							Optional[int]:




   """simple docstring"""







   _A          =      self.tokenizer_class(self.vocab_file   )

   _A          =      tokenizer.tokenize('''UNwant\u00E9d,running'''   )
   self.assertListEqual(a		,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )		,      [9, 6, 7, 1_2, 1_0, 1_1]   )
 def lowercase_						(  self   )		->							Union[str, Any]:




   """simple docstring"""







   if not self.test_rust_tokenizer:
     return

   _A          =      self.get_tokenizer()
   _A          =      self.get_rust_tokenizer()

   _A          =      '''UNwant\u00E9d,running'''

   _A          =      tokenizer.tokenize(a   )
   _A          =      rust_tokenizer.tokenize(a   )
   self.assertListEqual(a		,      a   )

   _A          =      tokenizer.encode(a		,      add_special_tokens=a   )
   _A          =      rust_tokenizer.encode(a		,      add_special_tokens=a   )
   self.assertListEqual(a		,      a   )

   _A          =      self.get_rust_tokenizer()
   _A          =      tokenizer.encode(a   )
   _A          =      rust_tokenizer.encode(a   )
   self.assertListEqual(a		,      a   )

   # With lower casing
   _A          =      self.get_tokenizer(do_lower_case=a   )
   _A          =      self.get_rust_tokenizer(do_lower_case=a   )

   _A          =      '''UNwant\u00E9d,running'''

   _A          =      tokenizer.tokenize(a   )
   _A          =      rust_tokenizer.tokenize(a   )
   self.assertListEqual(a		,      a   )

   _A          =      tokenizer.encode(a		,      add_special_tokens=a   )
   _A          =      rust_tokenizer.encode(a		,      add_special_tokens=a   )
   self.assertListEqual(a		,      a   )

   _A          =      self.get_rust_tokenizer()
   _A          =      tokenizer.encode(a   )
   _A          =      rust_tokenizer.encode(a   )
   self.assertListEqual(a		,      a   )
 def lowercase_						(  self   )		->							List[Any]:




   """simple docstring"""







   _A          =      BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )		,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )
 def lowercase_						(  self   )		->							List[Any]:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
 def lowercase_						(  self   )		->							Dict:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''h\u00E9llo''']   )
 def lowercase_						(  self   )		->							Optional[Any]:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
 def lowercase_						(  self   )		->							Optional[Any]:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )		,      ['''hello''']   )
 def lowercase_						(  self   )		->							List[Any]:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
 def lowercase_						(  self   )		->							int:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
 def lowercase_						(  self   )		->							List[str]:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a		,      strip_accents=a   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )		,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )
 def lowercase_						(  self   )		->							Tuple:




   """simple docstring"""







   _A          =      BasicTokenizer(do_lower_case=a		,      never_split=['''[UNK]''']   )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )
 def lowercase_						(  self   )		->							Tuple:




   """simple docstring"""







   _A          =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

   _A          =      {}
   for i, token in enumerate(a   ):
     _A          =      i
   _A          =      WordpieceTokenizer(vocab=a		,      unk_token='''[UNK]'''   )

   self.assertListEqual(tokenizer.tokenize(''''''   )		,      []   )

   self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )		,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

   self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )		,      ['''[UNK]''', '''runn''', '''##ing''']   )
 def lowercase_						(  self   )		->							Tuple:




   """simple docstring"""







   self.assertTrue(_is_whitespace(''' '''   )   )
   self.assertTrue(_is_whitespace('''\t'''   )   )
   self.assertTrue(_is_whitespace('''\r'''   )   )
   self.assertTrue(_is_whitespace('''\n'''   )   )
   self.assertTrue(_is_whitespace('''\u00A0'''   )   )

   self.assertFalse(_is_whitespace('''A'''   )   )
   self.assertFalse(_is_whitespace('''-'''   )   )
 def lowercase_						(  self   )		->							Optional[int]:




   """simple docstring"""







   self.assertTrue(_is_control('''\u0005'''   )   )

   self.assertFalse(_is_control('''A'''   )   )
   self.assertFalse(_is_control(''' '''   )   )
   self.assertFalse(_is_control('''\t'''   )   )
   self.assertFalse(_is_control('''\r'''   )   )
 def lowercase_						(  self   )		->							int:




   """simple docstring"""







   self.assertTrue(_is_punctuation('''-'''   )   )
   self.assertTrue(_is_punctuation('''$'''   )   )
   self.assertTrue(_is_punctuation('''`'''   )   )
   self.assertTrue(_is_punctuation('''.'''   )   )

   self.assertFalse(_is_punctuation('''A'''   )   )
   self.assertFalse(_is_punctuation(''' '''   )   )
 def lowercase_						(  self   )		->							Union[str, Any]:




   """simple docstring"""







   _A          =      self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]   )

   if self.test_rust_tokenizer:
     _A          =      self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]   )
 @slow
 def lowercase_						(  self   )		->							Dict:




   """simple docstring"""







   _A          =      self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

   _A          =      tokenizer.encode('''sequence builders'''		,      add_special_tokens=a   )
   _A          =      tokenizer.encode('''multi-sequence build'''		,      add_special_tokens=a   )

   _A          =      tokenizer.build_inputs_with_special_tokens(a   )
   _A          =      tokenizer.build_inputs_with_special_tokens(a		,      a   )

   assert encoded_sentence == [1_0_1] + text + [1_0_2]
   assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
 def lowercase_						(  self   )		->							str:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
       _A          =      self.rust_tokenizer_class.from_pretrained(a		,      **a   )

       _A          =      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
       _A          =      tokenizer_r.encode_plus(
           a		,      return_attention_mask=a		,      return_token_type_ids=a		,      return_offsets_mapping=a		,      add_special_tokens=a		,      )

       _A          =      tokenizer_r.do_lower_case if hasattr(a		,      '''do_lower_case'''   ) else False
       _A          =      (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''A'''),
               ((1, 2), ''','''),
               ((3, 5), '''na'''),
               ((5, 6), '''##ï'''),
               ((6, 8), '''##ve'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''Allen'''),
               ((2_1, 2_3), '''##NL'''),
               ((2_3, 2_4), '''##P'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''a'''),
               ((1, 2), ''','''),
               ((3, 8), '''naive'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''allen'''),
               ((2_1, 2_3), '''##nl'''),
               ((2_3, 2_4), '''##p'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]		,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
       self.assertEqual([e[0] for e in expected_results]		,      tokens['''offset_mapping''']   )



 @slow
 def lowercase_						(  self   )		->							Optional[Any]:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
       _A          =      self.rust_tokenizer_class.from_pretrained(a		,      **a   )
       _A          =      self.tokenizer_class.from_pretrained(a		,      **a   )
       _A          =      [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

       _A          =      tokenizer_r.batch_encode_candidates(a		,      max_length=1_0		,      return_tensors='''np'''   )
       _A          =      tokenizer_p.batch_encode_candidates(a		,      max_length=1_0		,      return_tensors='''np'''   )

       _A          =      (2, 2, 1_0)

       self.assertEqual(encoded_sentence_r['''input_ids'''].shape		,      a   )
       self.assertEqual(encoded_sentence_r['''attention_mask'''].shape		,      a   )
       self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape		,      a   )

       self.assertEqual(encoded_sentence_p['''input_ids'''].shape		,      a   )
       self.assertEqual(encoded_sentence_p['''attention_mask'''].shape		,      a   )
       self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape		,      a   )




import math
from timeit import timeit




def       UpperCAmelCase__	(							__snake_case							)				->	int:
  _A          =      0
  _A          =      abs(__snake_case							)
  while True:
    _A          =      n // 10
    digits += 1
    if n == 0:
      break
  return digits




def       UpperCAmelCase__	(							__snake_case							)				->	int:
  return 1 if n == 0 else math.floor(math.log(abs(__snake_case							)  ,			10							) + 1							)




def       UpperCAmelCase__	(							__snake_case							)				->	int:
  return len(str(abs(__snake_case							)							)							)




def       UpperCAmelCase__	(							)				->	None:
  from collections.abc import Callable

  def benchmark_a_function(__snake_case  ,			__snake_case							) -> None:
    _A          =      F'''{func.__name__}({value})'''
    _A          =      timeit(F'''__main__.{call}'''  ,			setup='''import __main__'''							)
    print(F'''{call}: {func(__snake_case							)} -- {timing} seconds'''							)

  for value in (262_144, 1_125_899_906_842_624, 1_267_650_600_228_229_401_496_703_205_376):
    for func in (num_digits, num_digits_fast, num_digits_faster):
      benchmark_a_function(__snake_case  ,			__snake_case							)
    print()


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()