






"""simple docstring"""



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 	a					(       a_,    unittest.TestCase     ):
    UpperCAmelCase_			:					Tuple     					=RealmTokenizer
    UpperCAmelCase_			:					List[Any]     					=RealmTokenizerFast
    UpperCAmelCase_			:					int     					=True
    UpperCAmelCase_			:					Union[str, Any]     					=True
    UpperCAmelCase_			:					List[str]     					=filter_non_english







    def       UpperCamelCase_				(      self	):
      super().setUp()

      lowercase							   =	[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowercase							   =	os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES['vocab_file']	)
      with open(self.vocab_file							,		'w'							,		encoding='utf-8'	) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      lowercase							   =	'UNwant\u00E9d,running'
      lowercase							   =	'unwanted, running'
      return input_text, output_text







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.tokenizer_class(self.vocab_file	)

      lowercase							   =	tokenizer.tokenize('UNwant\u00E9d,running'	)
      self.assertListEqual(_lowerCamelCase							,		['un', '##want', '##ed', ',', 'runn', '##ing']	)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase	)							,		[9, 6, 7, 1_2, 1_0, 1_1]	)







    def       UpperCamelCase_				(      self	):
      if not self.test_rust_tokenizer:
        return

      lowercase							   =	self.get_tokenizer()
      lowercase							   =	self.get_rust_tokenizer()

      lowercase							   =	'UNwant\u00E9d,running'

      lowercase							   =	tokenizer.tokenize(_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.tokenize(_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)

      lowercase							   =	tokenizer.encode(_lowerCamelCase							,		add_special_tokens=_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.encode(_lowerCamelCase							,		add_special_tokens=_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)

      lowercase							   =	self.get_rust_tokenizer()
      lowercase							   =	tokenizer.encode(_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.encode(_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)

      # With lower casing
      lowercase							   =	self.get_tokenizer(do_lower_case=_lowerCamelCase	)
      lowercase							   =	self.get_rust_tokenizer(do_lower_case=_lowerCamelCase	)

      lowercase							   =	'UNwant\u00E9d,running'

      lowercase							   =	tokenizer.tokenize(_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.tokenize(_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)

      lowercase							   =	tokenizer.encode(_lowerCamelCase							,		add_special_tokens=_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.encode(_lowerCamelCase							,		add_special_tokens=_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)

      lowercase							   =	self.get_rust_tokenizer()
      lowercase							   =	tokenizer.encode(_lowerCamelCase	)
      lowercase							   =	rust_tokenizer.encode(_lowerCamelCase	)
      self.assertListEqual(_lowerCamelCase							,		_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'	)							,		['ah', '\u535A', '\u63A8', 'zz']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)							,		['hello', '!', 'how', 'are', 'you', '?']	)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)							,		['hello']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase							,		strip_accents=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)							,		['hällo', '!', 'how', 'are', 'you', '?']	)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)							,		['h\u00E9llo']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase							,		strip_accents=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)							,		['hallo', '!', 'how', 'are', 'you', '?']	)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)							,		['hello']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)							,		['hallo', '!', 'how', 'are', 'you', '?']	)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)							,		['hello']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)							,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase							,		strip_accents=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)							,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase							,		strip_accents=_lowerCamelCase	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)							,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	BasicTokenizer(do_lower_case=_lowerCamelCase							,		never_split=['[UNK]']	)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'	)							,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowercase							   =	{}
      for i, token in enumerate(_lowerCamelCase	):
        lowercase							   =	i
      lowercase							   =	WordpieceTokenizer(vocab=_lowerCamelCase							,		unk_token='[UNK]'	)

      self.assertListEqual(tokenizer.tokenize(''	)							,		[]	)

      self.assertListEqual(tokenizer.tokenize('unwanted running'	)							,		['un', '##want', '##ed', 'runn', '##ing']	)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'	)							,		['[UNK]', 'runn', '##ing']	)







    def       UpperCamelCase_				(      self	):
      self.assertTrue(_is_whitespace(' '	)	)
      self.assertTrue(_is_whitespace('\t'	)	)
      self.assertTrue(_is_whitespace('\r'	)	)
      self.assertTrue(_is_whitespace('\n'	)	)
      self.assertTrue(_is_whitespace('\u00A0'	)	)

      self.assertFalse(_is_whitespace('A'	)	)
      self.assertFalse(_is_whitespace('-'	)	)







    def       UpperCamelCase_				(      self	):
      self.assertTrue(_is_control('\u0005'	)	)

      self.assertFalse(_is_control('A'	)	)
      self.assertFalse(_is_control(' '	)	)
      self.assertFalse(_is_control('\t'	)	)
      self.assertFalse(_is_control('\r'	)	)







    def       UpperCamelCase_				(      self	):
      self.assertTrue(_is_punctuation('-'	)	)
      self.assertTrue(_is_punctuation('$'	)	)
      self.assertTrue(_is_punctuation('`'	)	)
      self.assertTrue(_is_punctuation('.'	)	)

      self.assertFalse(_is_punctuation('A'	)	)
      self.assertFalse(_is_punctuation(' '	)	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_lowerCamelCase	) for t in ['Test', '\xad', 'test']]							,		[['[UNK]'], [], ['[UNK]']]	)

      if self.test_rust_tokenizer:
        lowercase							   =	self.get_rust_tokenizer()
        self.assertListEqual(
            [rust_tokenizer.tokenize(_lowerCamelCase	) for t in ['Test', '\xad', 'test']]							,		[['[UNK]'], [], ['[UNK]']]	)







    @slow
    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.tokenizer_class.from_pretrained('bert-base-uncased'	)

      lowercase							   =	tokenizer.encode('sequence builders'							,		add_special_tokens=_lowerCamelCase	)
      lowercase							   =	tokenizer.encode('multi-sequence build'							,		add_special_tokens=_lowerCamelCase	)

      lowercase							   =	tokenizer.build_inputs_with_special_tokens(_lowerCamelCase	)
      lowercase							   =	tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							,		_lowerCamelCase	)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







    def       UpperCamelCase_				(      self	):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
          lowercase							   =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,		**_lowerCamelCase	)

          lowercase							   =	F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
          lowercase							   =	tokenizer_r.encode_plus(
              _lowerCamelCase							,		return_attention_mask=_lowerCamelCase							,		return_token_type_ids=_lowerCamelCase							,		return_offsets_mapping=_lowerCamelCase							,		add_special_tokens=_lowerCamelCase							,		)

          lowercase							   =	tokenizer_r.do_lower_case if hasattr(_lowerCamelCase							,		'do_lower_case'	) else False
          lowercase							   =	(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 1_5), tokenizer_r.mask_token),
                  ((1_6, 2_1), 'Allen'),
                  ((2_1, 2_3), '##NL'),
                  ((2_3, 2_4), '##P'),
                  ((2_5, 3_3), 'sentence'),
                  ((3_3, 3_4), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 1_5), tokenizer_r.mask_token),
                  ((1_6, 2_1), 'allen'),
                  ((2_1, 2_3), '##nl'),
                  ((2_3, 2_4), '##p'),
                  ((2_5, 3_3), 'sentence'),
                  ((3_3, 3_4), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]							,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']	)	)
          self.assertEqual([e[0] for e in expected_results]							,		tokens['offset_mapping']	)







    @slow
    def       UpperCamelCase_				(      self	):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
          lowercase							   =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,		**_lowerCamelCase	)
          lowercase							   =	self.tokenizer_class.from_pretrained(_lowerCamelCase							,		**_lowerCamelCase	)
          lowercase							   =	[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

          lowercase							   =	tokenizer_r.batch_encode_candidates(_lowerCamelCase							,		max_length=1_0							,		return_tensors='np'	)
          lowercase							   =	tokenizer_p.batch_encode_candidates(_lowerCamelCase							,		max_length=1_0							,		return_tensors='np'	)

          lowercase							   =	(2, 2, 1_0)

          self.assertEqual(encoded_sentence_r['input_ids'].shape							,		_lowerCamelCase	)
          self.assertEqual(encoded_sentence_r['attention_mask'].shape							,		_lowerCamelCase	)
          self.assertEqual(encoded_sentence_r['token_type_ids'].shape							,		_lowerCamelCase	)

          self.assertEqual(encoded_sentence_p['input_ids'].shape							,		_lowerCamelCase	)
          self.assertEqual(encoded_sentence_p['attention_mask'].shape							,		_lowerCamelCase	)
          self.assertEqual(encoded_sentence_p['token_type_ids'].shape							,		_lowerCamelCase	)










"""simple docstring"""



_UpperCamelCase		:  str						=		1_0**8
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  int   ):





  '''simple docstring'''





  if n % 10 == 0:
    return False
  lowercase							   =	str(__snake_case   )
  return s == s[::-1]
def 						_SCREAMING_SNAKE_CASE			(   ):





  '''simple docstring'''





  lowercase							   =	set()
  lowercase							   =	1
  lowercase							   =	5
  while sum_squares < LIMIT:
    lowercase							   =	first_square + 1
    while sum_squares < LIMIT:
      if is_palindrome(__snake_case   ):
        answer.add(__snake_case   )
      last_square += 1
      sum_squares += last_square**2
    first_square += 1
    lowercase							   =	first_square**2 + (first_square + 1) ** 2

  return sum(__snake_case   )


if __name__ == "__main__":
    print(solution())


