


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 				lowerCamelCase__       (	lowerCAmelCase       ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__				      =      RealmTokenizer
     SCREAMING_SNAKE_CASE__				      =      RealmTokenizerFast
     SCREAMING_SNAKE_CASE__				      =      True
     SCREAMING_SNAKE_CASE__				      =      True
     SCREAMING_SNAKE_CASE__				      =      filter_non_english





     def 					__A (self	)					->       Optional[int]:
      super().setUp()

      _lowercase											=[
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      _lowercase											=os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES['''vocab_file''']	)
      with open(self.vocab_file		,					'''w'''		,					encoding='''utf-8'''	) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)





     def 					__A (self		,					UpperCAmelCase	)					->       Optional[int]:
      _lowercase											='''UNwant\u00E9d,running'''
      _lowercase											='''unwanted, running'''
      return input_text, output_text





     def 					__A (self	)					->       Tuple:
      _lowercase											=self.tokenizer_class(self.vocab_file	)

      _lowercase											=tokenizer.tokenize('''UNwant\u00E9d,running'''	)
      self.assertListEqual(UpperCAmelCase		,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase	)		,					[9, 6, 7, 1_2, 1_0, 1_1]	)





     def 					__A (self	)					->       Any:
      if not self.test_rust_tokenizer:
       return

      _lowercase											=self.get_tokenizer()
      _lowercase											=self.get_rust_tokenizer()

      _lowercase											='''UNwant\u00E9d,running'''

      _lowercase											=tokenizer.tokenize(UpperCAmelCase	)
      _lowercase											=rust_tokenizer.tokenize(UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)

      _lowercase											=tokenizer.encode(UpperCAmelCase		,					add_special_tokens=UpperCAmelCase	)
      _lowercase											=rust_tokenizer.encode(UpperCAmelCase		,					add_special_tokens=UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)

      _lowercase											=self.get_rust_tokenizer()
      _lowercase											=tokenizer.encode(UpperCAmelCase	)
      _lowercase											=rust_tokenizer.encode(UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)

      # With lower casing
      _lowercase											=self.get_tokenizer(do_lower_case=UpperCAmelCase	)
      _lowercase											=self.get_rust_tokenizer(do_lower_case=UpperCAmelCase	)

      _lowercase											='''UNwant\u00E9d,running'''

      _lowercase											=tokenizer.tokenize(UpperCAmelCase	)
      _lowercase											=rust_tokenizer.tokenize(UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)

      _lowercase											=tokenizer.encode(UpperCAmelCase		,					add_special_tokens=UpperCAmelCase	)
      _lowercase											=rust_tokenizer.encode(UpperCAmelCase		,					add_special_tokens=UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)

      _lowercase											=self.get_rust_tokenizer()
      _lowercase											=tokenizer.encode(UpperCAmelCase	)
      _lowercase											=rust_tokenizer.encode(UpperCAmelCase	)
      self.assertListEqual(UpperCAmelCase		,					UpperCAmelCase	)





     def 					__A (self	)					->       List[Any]:
      _lowercase											=BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)		,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)





     def 					__A (self	)					->       Optional[int]:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)		,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)		,					['''hello''']	)





     def 					__A (self	)					->       Dict:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase		,					strip_accents=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)		,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)		,					['''h\u00E9llo''']	)





     def 					__A (self	)					->       Tuple:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase		,					strip_accents=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)		,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)		,					['''hello''']	)





     def 					__A (self	)					->       int:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)		,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)		,					['''hello''']	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)		,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase		,					strip_accents=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)		,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





     def 					__A (self	)					->       int:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase		,					strip_accents=UpperCAmelCase	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)		,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





     def 					__A (self	)					->       int:
      _lowercase											=BasicTokenizer(do_lower_case=UpperCAmelCase		,					never_split=['''[UNK]''']	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)		,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)





     def 					__A (self	)					->       Optional[Any]:
      _lowercase											=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      _lowercase											={}
      for i, token in enumerate(UpperCAmelCase	):
       _lowercase											=i
      _lowercase											=WordpieceTokenizer(vocab=UpperCAmelCase		,					unk_token='''[UNK]'''	)

      self.assertListEqual(tokenizer.tokenize(''''''	)		,					[]	)

      self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)		,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

      self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)		,					['''[UNK]''', '''runn''', '''##ing''']	)





     def 					__A (self	)					->       Optional[Any]:
      self.assertTrue(_is_whitespace(''' '''	)	)
      self.assertTrue(_is_whitespace('''\t'''	)	)
      self.assertTrue(_is_whitespace('''\r'''	)	)
      self.assertTrue(_is_whitespace('''\n'''	)	)
      self.assertTrue(_is_whitespace('''\u00A0'''	)	)

      self.assertFalse(_is_whitespace('''A'''	)	)
      self.assertFalse(_is_whitespace('''-'''	)	)





     def 					__A (self	)					->       List[Any]:
      self.assertTrue(_is_control('''\u0005'''	)	)

      self.assertFalse(_is_control('''A'''	)	)
      self.assertFalse(_is_control(''' '''	)	)
      self.assertFalse(_is_control('''\t'''	)	)
      self.assertFalse(_is_control('''\r'''	)	)





     def 					__A (self	)					->       List[str]:
      self.assertTrue(_is_punctuation('''-'''	)	)
      self.assertTrue(_is_punctuation('''$'''	)	)
      self.assertTrue(_is_punctuation('''`'''	)	)
      self.assertTrue(_is_punctuation('''.'''	)	)

      self.assertFalse(_is_punctuation('''A'''	)	)
      self.assertFalse(_is_punctuation(''' '''	)	)





     def 					__A (self	)					->       Any:
      _lowercase											=self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(UpperCAmelCase	) for t in ['''Test''', '''\xad''', '''test''']]		,					[['''[UNK]'''], [], ['''[UNK]''']]	)

      if self.test_rust_tokenizer:
       _lowercase											=self.get_rust_tokenizer()
       self.assertListEqual(
           [rust_tokenizer.tokenize(UpperCAmelCase	) for t in ['''Test''', '''\xad''', '''test''']]		,					[['''[UNK]'''], [], ['''[UNK]''']]	)





     @slow
     def 					__A (self	)					->       Any:
      _lowercase											=self.tokenizer_class.from_pretrained('''bert-base-uncased'''	)

      _lowercase											=tokenizer.encode('''sequence builders'''		,					add_special_tokens=UpperCAmelCase	)
      _lowercase											=tokenizer.encode('''multi-sequence build'''		,					add_special_tokens=UpperCAmelCase	)

      _lowercase											=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase	)
      _lowercase											=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase		,					UpperCAmelCase	)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





     def 					__A (self	)					->       Any:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
        _lowercase											=self.rust_tokenizer_class.from_pretrained(UpperCAmelCase		,					**UpperCAmelCase	)

        _lowercase											=f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
        _lowercase											=tokenizer_r.encode_plus(
            UpperCAmelCase		,					return_attention_mask=UpperCAmelCase		,					return_token_type_ids=UpperCAmelCase		,					return_offsets_mapping=UpperCAmelCase		,					add_special_tokens=UpperCAmelCase		,					)

        _lowercase											=tokenizer_r.do_lower_case if hasattr(UpperCAmelCase		,					'''do_lower_case'''	) else False
        _lowercase											=(
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''A'''),
                ((1, 2), ''','''),
                ((3, 5), '''na'''),
                ((5, 6), '''##ï'''),
                ((6, 8), '''##ve'''),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), '''Allen'''),
                ((2_1, 2_3), '''##NL'''),
                ((2_3, 2_4), '''##P'''),
                ((2_5, 3_3), '''sentence'''),
                ((3_3, 3_4), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''a'''),
                ((1, 2), ''','''),
                ((3, 8), '''naive'''),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), '''allen'''),
                ((2_1, 2_3), '''##nl'''),
                ((2_3, 2_4), '''##p'''),
                ((2_5, 3_3), '''sentence'''),
                ((3_3, 3_4), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )

        self.assertEqual(
            [e[1] for e in expected_results]		,					tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
        self.assertEqual([e[0] for e in expected_results]		,					tokens['''offset_mapping''']	)





     @slow
     def 					__A (self	)					->       Optional[int]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
        _lowercase											=self.rust_tokenizer_class.from_pretrained(UpperCAmelCase		,					**UpperCAmelCase	)
        _lowercase											=self.tokenizer_class.from_pretrained(UpperCAmelCase		,					**UpperCAmelCase	)
        _lowercase											=[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

        _lowercase											=tokenizer_r.batch_encode_candidates(UpperCAmelCase		,					max_length=1_0		,					return_tensors='''np'''	)
        _lowercase											=tokenizer_p.batch_encode_candidates(UpperCAmelCase		,					max_length=1_0		,					return_tensors='''np'''	)

        _lowercase											=(2, 2, 1_0)

        self.assertEqual(encoded_sentence_r['''input_ids'''].shape		,					UpperCAmelCase	)
        self.assertEqual(encoded_sentence_r['''attention_mask'''].shape		,					UpperCAmelCase	)
        self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape		,					UpperCAmelCase	)

        self.assertEqual(encoded_sentence_p['''input_ids'''].shape		,					UpperCAmelCase	)
        self.assertEqual(encoded_sentence_p['''attention_mask'''].shape		,					UpperCAmelCase	)
        self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape		,					UpperCAmelCase	)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__            =							logging.get_logger(__name__)

UpperCAmelCase__            =							{
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}







class 				lowerCamelCase__       (	lowerCAmelCase):
     SCREAMING_SNAKE_CASE__				      =      '''qdqbert'''





     def __init__(self		,					UpperCAmelCase=3_0_5_2_2		,					UpperCAmelCase=7_6_8		,					UpperCAmelCase=1_2		,					UpperCAmelCase=1_2		,					UpperCAmelCase=3_0_7_2		,					UpperCAmelCase="gelu"		,					UpperCAmelCase=0.1		,					UpperCAmelCase=0.1		,					UpperCAmelCase=5_1_2		,					UpperCAmelCase=2		,					UpperCAmelCase=0.02		,					UpperCAmelCase=1e-12		,					UpperCAmelCase=True		,					UpperCAmelCase=1		,					UpperCAmelCase=0		,					UpperCAmelCase=2		,					**UpperCAmelCase		,					)					->       str:
      super().__init__(pad_token_id=UpperCAmelCase		,					bos_token_id=UpperCAmelCase		,					eos_token_id=UpperCAmelCase		,					**UpperCAmelCase	)

      _lowercase											=vocab_size
      _lowercase											=max_position_embeddings
      _lowercase											=hidden_size
      _lowercase											=num_hidden_layers
      _lowercase											=num_attention_heads
      _lowercase											=intermediate_size
      _lowercase											=hidden_act
      _lowercase											=hidden_dropout_prob
      _lowercase											=attention_probs_dropout_prob
      _lowercase											=initializer_range
      _lowercase											=type_vocab_size
      _lowercase											=layer_norm_eps
      _lowercase											=use_cache

