




'''simple docstring'''







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

A			   =logging.get_logger(__name__)

A			   ={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

A			   ={
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

A			   ={
    'microsoft/deberta-base': 5_12,
    'microsoft/deberta-large': 5_12,
    'microsoft/deberta-xlarge': 5_12,
    'microsoft/deberta-base-mnli': 5_12,
    'microsoft/deberta-large-mnli': 5_12,
    'microsoft/deberta-xlarge-mnli': 5_12,
}

A			   ={
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}





def 						snake_case_							():
    UpperCAmelCase      	=		(
        list(range(ord('''!'''					)   ,       ord('''~'''					) + 1					)					) + list(range(ord('''¡'''					)   ,       ord('''¬'''					) + 1					)					) + list(range(ord('''®'''					)   ,       ord('''ÿ'''					) + 1					)					)
    )
    UpperCAmelCase      	=		bs[:]
    UpperCAmelCase      	=		0
    for b in range(2**8					):
        if b not in bs:
            bs.append(_a					)
            cs.append(2**8 + n					)
            n += 1
    UpperCAmelCase      	=		[chr(_a					) for n in cs]
    return dict(zip(_a   ,       _a					)					)





def 						snake_case_							(_a     :							int					):
    UpperCAmelCase      	=		set()
    UpperCAmelCase      	=		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)					)
        UpperCAmelCase      	=		char
    return pairs







class       _a  (     __a			):
       __a    :					List[Any]           = VOCAB_FILES_NAMES
       __a    :					Tuple           = PRETRAINED_VOCAB_FILES_MAP
       __a    :					Tuple           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __a    :					Union[str, Any]           = ["""input_ids""", """attention_mask""", """token_type_ids"""]
       def __init__(     self						:     int       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Optional[int]       ,    lowercase						:     Tuple="replace"       ,    lowercase						:     Optional[int]="[CLS]"       ,    lowercase						:     List[Any]="[SEP]"       ,    lowercase						:     Tuple="[SEP]"       ,    lowercase						:     Dict="[CLS]"       ,    lowercase						:     Tuple="[UNK]"       ,    lowercase						:     List[Any]="[PAD]"       ,    lowercase						:     Union[str, Any]="[MASK]"       ,    lowercase						:     Optional[Any]=False       ,    lowercase						:     int=False       ,    **lowercase						:     str       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else bos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else eos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else sep_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else cls_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else unk_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else mask_token

           super().__init__(
               errors=lowercase       ,    bos_token=lowercase       ,    eos_token=lowercase       ,    unk_token=lowercase       ,    sep_token=lowercase       ,    cls_token=lowercase       ,    pad_token=lowercase       ,    mask_token=lowercase       ,    add_prefix_space=lowercase       ,    add_bos_token=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		add_bos_token

           with open(lowercase       ,    encoding='''utf-8'''    ) as vocab_handle:
               UpperCAmelCase      	=		json.load(lowercase    )
           UpperCAmelCase      	=		{v: k for k, v in self.encoder.items()}
           UpperCAmelCase      	=		errors  # how to handle errors in decoding
           UpperCAmelCase      	=		bytes_to_unicode()
           UpperCAmelCase      	=		{v: k for k, v in self.byte_encoder.items()}
           with open(lowercase       ,    encoding='''utf-8'''    ) as merges_handle:
               UpperCAmelCase      	=		merges_handle.read().split('''\n'''    )[1:-1]
           UpperCAmelCase      	=		[tuple(merge.split()    ) for merge in bpe_merges]
           UpperCAmelCase      	=		dict(zip(lowercase       ,    range(len(lowercase    )    )    )    )
           UpperCAmelCase      	=		{}
           UpperCAmelCase      	=		add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           UpperCAmelCase      	=		re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )
       @property
       # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           return len(self.encoder    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           return dict(self.encoder       ,    **self.added_tokens_encoder    )
       def    A				(     self						:     str       ,    lowercase						:     Optional[Any]    ):



           '''simple docstring'''

           if token in self.cache:
               return self.cache[token]
           UpperCAmelCase      	=		tuple(lowercase    )
           UpperCAmelCase      	=		get_pairs(lowercase    )

           if not pairs:
               return token

           while True:
               UpperCAmelCase      	=		min(lowercase       ,    key=lambda lowercase    : self.bpe_ranks.get(lowercase       ,    float('''inf'''    )    )    )
               if bigram not in self.bpe_ranks:
                   break
               UpperCAmelCase	,					UpperCAmelCase      	=		bigram
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		0
               while i < len(lowercase    ):
                   try:
                       UpperCAmelCase      	=		word.index(lowercase       ,    lowercase    )
                   except ValueError:
                       new_word.extend(word[i:]    )
                       break
                   else:
                       new_word.extend(word[i:j]    )
                       UpperCAmelCase      	=		j

                   if word[i] == first and i < len(lowercase    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                   else:
                       new_word.append(word[i]    )
                       i += 1
               UpperCAmelCase      	=		tuple(lowercase    )
               UpperCAmelCase      	=		new_word
               if len(lowercase    ) == 1:
                   break
               else:
                   UpperCAmelCase      	=		get_pairs(lowercase    )
           UpperCAmelCase      	=		''' '''.join(lowercase    )
           UpperCAmelCase      	=		word
           return word
       def    A				(     self						:     List[str]       ,    lowercase						:     List[int]       ,    lowercase						:     Optional[List[int]] = None    ):



           '''simple docstring'''

           if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           UpperCAmelCase      	=		[self.cls_token_id]
           UpperCAmelCase      	=		[self.sep_token_id]
           return cls + token_ids_a + sep + token_ids_a + sep
       def    A				(     self						:     int       ,    lowercase						:     List[int]       ,    lowercase						:     Optional[List[int]] = None       ,    lowercase						:     bool = False    ):



           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowercase       ,    token_ids_a=lowercase       ,    already_has_special_tokens=lowercase    )

           if token_ids_a is None:
               return [1] + ([0] * len(lowercase    )) + [1]
           return [1] + ([0] * len(lowercase    )) + [1] + ([0] * len(lowercase    )) + [1]
       def    A				(     self						:     int       ,    lowercase						:     List[int]       ,    lowercase						:     Optional[List[int]] = None    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[self.sep_token_id]
           UpperCAmelCase      	=		[self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]
       def    A				(     self						:     int       ,    lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for token in re.findall(self.pat       ,    lowercase    ):
               UpperCAmelCase      	=		''''''.join(
                   self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase    ).split(''' '''    )    )
           return bpe_tokens
       def    A				(     self						:     int       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           return self.encoder.get(lowercase       ,    self.encoder.get(self.unk_token    )    )
       def    A				(     self						:     List[Any]       ,    lowercase						:     List[Any]    ):



           '''simple docstring'''

           return self.decoder.get(lowercase    )
       def    A				(     self						:     int       ,    lowercase						:     List[str]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		''''''.join(lowercase    )
           UpperCAmelCase      	=		bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''       ,    errors=self.errors    )
           return text
       def    A				(     self						:     str       ,    lowercase						:     str       ,    lowercase						:     Optional[str] = None    ):



           '''simple docstring'''

           if not os.path.isdir(lowercase    ):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
               return
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as f:
               f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=lowercase       ,    ensure_ascii=lowercase    ) + '''\n'''    )

           UpperCAmelCase      	=		0
           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as writer:
               writer.write('''#version: 0.2\n'''    )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda lowercase    : kv[1]    ):
                   if index != token_index:
                       logger.warning(
                           f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                           ''' Please check that the tokenizer is not corrupted!'''    )
                       UpperCAmelCase      	=		token_index
                   writer.write(''' '''.join(lowercase    ) + '''\n'''    )
                   index += 1

           return vocab_file, merge_file
       def    A				(     self						:     Optional[int]       ,    lowercase						:     str       ,    lowercase						:     Dict=False       ,    **lowercase						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		kwargs.pop('''add_prefix_space'''       ,    self.add_prefix_space    )
           if (is_split_into_words or add_prefix_space) and (len(lowercase    ) > 0 and not text[0].isspace()):
               UpperCAmelCase      	=		''' ''' + text
           return (text, kwargs)







       def    A				(     self						:     int       ,    lowercase						:     "Conversation"    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(lowercase       ,    add_special_tokens=lowercase    ) + [self.eos_token_id]    )
           if len(lowercase    ) > self.model_max_length:
               UpperCAmelCase      	=		input_ids[-self.model_max_length :]
           return input_ids










'''simple docstring'''







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					Dict           = RealmTokenizer
       __a    :					Dict           = RealmTokenizerFast
       __a    :					Union[str, Any]           = True
       __a    :					Optional[int]           = True
       __a    :					int           = filter_non_english
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           super().setUp()

           UpperCAmelCase      	=		[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     int       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''
           UpperCAmelCase      	=		'''unwanted, running'''
           return input_text, output_text
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''UNwant\u00E9d,running'''    )
           self.assertListEqual(lowercase       ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [9, 6, 7, 12, 10, 11]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase      	=		self.get_tokenizer()
           UpperCAmelCase      	=		self.get_rust_tokenizer()

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           # With lower casing
           UpperCAmelCase      	=		self.get_tokenizer(do_lower_case=lowercase    )
           UpperCAmelCase      	=		self.get_rust_tokenizer(do_lower_case=lowercase    )

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )       ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''h\u00E9llo''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    never_split=['''[UNK]''']    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           UpperCAmelCase      	=		{}
           for i, token in enumerate(lowercase    ):
               UpperCAmelCase      	=		i
           UpperCAmelCase      	=		WordpieceTokenizer(vocab=lowercase       ,    unk_token='''[UNK]'''    )

           self.assertListEqual(tokenizer.tokenize(''''''    )       ,    []    )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )       ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )       ,    ['''[UNK]''', '''runn''', '''##ing''']    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''    )    )
           self.assertTrue(_is_whitespace('''\t'''    )    )
           self.assertTrue(_is_whitespace('''\r'''    )    )
           self.assertTrue(_is_whitespace('''\n'''    )    )
           self.assertTrue(_is_whitespace('''\u00A0'''    )    )

           self.assertFalse(_is_whitespace('''A'''    )    )
           self.assertFalse(_is_whitespace('''-'''    )    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''    )    )

           self.assertFalse(_is_control('''A'''    )    )
           self.assertFalse(_is_control(''' '''    )    )
           self.assertFalse(_is_control('''\t'''    )    )
           self.assertFalse(_is_control('''\r'''    )    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''    )    )
           self.assertTrue(_is_punctuation('''$'''    )    )
           self.assertTrue(_is_punctuation('''`'''    )    )
           self.assertTrue(_is_punctuation('''.'''    )    )

           self.assertFalse(_is_punctuation('''A'''    )    )
           self.assertFalse(_is_punctuation(''' '''    )    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )

           if self.test_rust_tokenizer:
               UpperCAmelCase      	=		self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )
       @slow
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''    )

           UpperCAmelCase      	=		tokenizer.encode('''sequence builders'''       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		tokenizer.encode('''multi-sequence build'''       ,    add_special_tokens=lowercase    )

           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase    )
           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase       ,    lowercase    )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]
       def    A				(     self						:     int    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )

                   UpperCAmelCase      	=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                   UpperCAmelCase      	=		tokenizer_r.encode_plus(
                       lowercase       ,    return_attention_mask=lowercase       ,    return_token_type_ids=lowercase       ,    return_offsets_mapping=lowercase       ,    add_special_tokens=lowercase       ,    )

                   UpperCAmelCase      	=		tokenizer_r.do_lower_case if hasattr(lowercase       ,    '''do_lower_case'''    ) else False
                   UpperCAmelCase      	=		(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
                   self.assertEqual([e[0] for e in expected_results]       ,    tokens['''offset_mapping''']    )







       @slow
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		self.tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   UpperCAmelCase      	=		tokenizer_r.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )
                   UpperCAmelCase      	=		tokenizer_p.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )

                   UpperCAmelCase      	=		(2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape       ,    lowercase    )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape       ,    lowercase    )




