




'''simple docstring'''







import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
   import importlib.resources as pkg_resources
except ImportError:
   # Try backported to PY<37 `importlib_resources`.
   import importlib_resources as pkg_resources

from . import resources


A			   ='''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
A			   =f"""{BASE_REF_URL}/{__file__}"""
A			   =logging.getLogger(__name__)





def 						snake_case_							(_a     :							List[Any]					):
    UpperCAmelCase      	=		pkg_resources.read_text(_a   ,       _a					)
    return yaml.safe_load(_a					), F"{BASE_REF_URL}/resources/{resource}"


A			   =load_yaml_resource('readme_structure.yaml')

A			   =[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
A			   =Tuple[dict, List[str], List[str]]


class       _a  :
       def __init__(     self						:     Tuple       ,    lowercase						:     Any       ,    lowercase						:     List[str]       ,    lowercase						:     int = None       ,    lowercase						:     Any = False    ):



           '''simple docstring'''

           UpperCAmelCase      	=		name
           UpperCAmelCase      	=		level
           UpperCAmelCase      	=		lines
           UpperCAmelCase      	=		''
           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		{}
           UpperCAmelCase      	=		[]
           UpperCAmelCase      	=		[]
           if self.lines is not None:
               self.parse(suppress_parsing_errors=_A    )
       def    A				(     self						:     Optional[int]       ,    lowercase						:     Any = False    ):



           '''simple docstring'''

           UpperCAmelCase      	=		''
           UpperCAmelCase      	=		[]
           UpperCAmelCase      	=		False
           for line in self.lines:
               if line.strip(''' \n'''    ) == "":
                   continue
               elif line.strip(''' \n'''    )[:3] == "```":
                   UpperCAmelCase      	=		not code_start
               elif line.split()[0] == self.level + "#" and not code_start:
                   if current_sub_level != "":
                       UpperCAmelCase      	=		Section(_A       ,    self.level + '''#'''       ,    _A    )
                       UpperCAmelCase      	=		[]
                   else:
                       if current_lines != []:
                           self.text += "".join(_A    ).strip()
                           if self.text != "" and self.text not in FILLER_TEXT:
                               UpperCAmelCase      	=		False
                           UpperCAmelCase      	=		[]

                   UpperCAmelCase      	=		' '.join(line.split()[1:]    ).strip(''' \n'''    )
               else:
                   current_lines.append(_A    )
           else:
               if current_sub_level != "":
                   if current_sub_level in self.content:
                       self.parsing_error_list.append(
                           f"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."    )
                   UpperCAmelCase      	=		Section(_A       ,    self.level + '''#'''       ,    _A    )
               else:
                   if current_lines != []:
                       self.text += "".join(_A    ).strip()
                       if self.text != "" and self.text not in FILLER_TEXT:
                           UpperCAmelCase      	=		False

           if self.level == "" and not suppress_parsing_errors:
               if self.parsing_error_list != [] or self.parsing_warning_list != []:
                   UpperCAmelCase      	=		'\n'.join('''-\t''' + x for x in self.parsing_error_list + self.parsing_warning_list    )
                   UpperCAmelCase      	=		f"The following issues were found while parsing the README at `{self.name}`:\n" + errors
                   raise ValueError(_A    )
       def    A				(     self						:     Union[str, Any]       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           UpperCAmelCase      	=		[]
           if structure["allow_empty"] is False:
               # If content is expected
               if self.is_empty_text and self.content == {}:
                   # If no content is found, mention it in the error_list
                   error_list.append(f"Expected some content in section `{self.name}` but it is empty."    )

           if structure["allow_empty_text"] is False:
               # If some text is expected
               if self.is_empty_text:
                   # If no text is found, mention it in the error_list
                   error_list.append(
                       f"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."    )
        # Subsections Validation
           if structure["subsections"] is not None:
               # If subsections are expected
               if self.content == {}:
                   # If no subsections are present
                   UpperCAmelCase      	=		[subsection['name'] for subsection in structure['subsections']]
                   # Mention the expected values in the error_list
                   error_list.append(
                       f"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]    )}. Found 'None'."    )
               else:
                   # If some subsections are present
                   UpperCAmelCase      	=		[subsection['name'] for subsection in structure['subsections']]
                   UpperCAmelCase      	=		False
                   for idx, name in enumerate(_A    ):
                       if name not in self.content:
                           # If the expected subsection is not present
                           error_list.append(f"Section `{self.name}` is missing subsection: `{name}`."    )
                           UpperCAmelCase      	=		True
                       else:
                           # If the subsection is present, validate subsection, return the result
                           # and concat the errors from subsection to section error_list

                           # Skip sublevel validation if current level is `###`
                           if self.level == "###":
                               continue
                           else:
                               UpperCAmelCase      	=		self.content[name].validate(
                                   structure['''subsections'''][idx]    )
                           error_list += subsec_error_list
                           warning_list += subsec_warning_list

                   if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                       for name in self.content:
                           if name not in structure_names:
                               # If an extra subsection is present
                               warning_list.append(
                                   f"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."    )
           if error_list:
               # If there are errors, do not return the dictionary as it is invalid
               return {}, error_list, warning_list
           else:
               return self.to_dict(), error_list, warning_list







       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           return {
               "name": self.name,
               "text": self.text,
               "is_empty_text": self.is_empty_text,
               "subsections": [value.to_dict() for value in self.content.values()],
           }






class       _a  (     snake_case__			):  # Level 0
       def __init__(     self						:     Union[str, Any]       ,    lowercase						:     List[str]       ,    lowercase						:     str       ,    lowercase						:     Any = None       ,    lowercase						:     Optional[int] = False    ):



           '''simple docstring'''

           super().__init__(name=_A       ,    level=''''''    )  # Not using lines here as we need to use a child class parse
           UpperCAmelCase      	=		structure
           UpperCAmelCase      	=		-2
           UpperCAmelCase      	=		0
           UpperCAmelCase      	=		lines
           if self.lines is not None:
               self.parse(suppress_parsing_errors=_A    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           if self.structure is None:
               UpperCAmelCase      	=		self._validate(_A    )
           else:
               UpperCAmelCase      	=		self._validate(self.structure    )
           if error_list != [] or warning_list != []:
               UpperCAmelCase      	=		'\n'.join(['''-\t''' + x for x in error_list + warning_list]    )
               UpperCAmelCase      	=		f"The following issues were found for the README at `{self.name}`:\n" + errors
               raise ValueError(_A    )
       @classmethod
       def    A				(     cls						:     Union[str, Any]       ,    lowercase						:     Dict       ,    lowercase						:     Union[str, Any] = None       ,    lowercase						:     List[str] = False    ):



           '''simple docstring'''

           with open(_A       ,    encoding='''utf-8'''    ) as f:
               UpperCAmelCase      	=		f.readlines()
           return cls(_A       ,    _A       ,    _A       ,    suppress_parsing_errors=_A    )
       @classmethod
       def    A				(     cls						:     str       ,    lowercase						:     List[str]       ,    lowercase						:     Any = None       ,    lowercase						:     Dict = "root"       ,    lowercase						:     int = False    ):



           '''simple docstring'''

           UpperCAmelCase      	=		string.split('''\n'''    )
           return cls(_A       ,    _A       ,    _A       ,    suppress_parsing_errors=_A    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     List[Any] = False    ):



           '''simple docstring'''

           UpperCAmelCase      	=		0

           for line in self.lines:
               self.yaml_tags_line_count += 1
               if line.strip(''' \n'''    ) == "---":
                   self.tag_count += 1
                   if self.tag_count == 2:
                       break
               line_count += 1
           if self.tag_count == 2:
               UpperCAmelCase      	=		self.lines[line_count + 1 :]  # Get the last + 1 th item.
           else:
               UpperCAmelCase      	=		self.lines[self.tag_count :]
           super().parse(suppress_parsing_errors=_A    )
       def __str__(     self						:     Optional[Any]    ):



           '''simple docstring'''

           return str(self.to_dict()    )







       def    A				(     self						:     Optional[Any]       ,    lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           UpperCAmelCase      	=		[]
           if self.yaml_tags_line_count == 0:
               warning_list.append('''Empty YAML markers are present in the README.'''    )
           elif self.tag_count == 0:
               warning_list.append('''No YAML markers are present in the README.'''    )
           elif self.tag_count == 1:
               warning_list.append('''Only the start of YAML tags present in the README.'''    )
           # Check how many first level sections are present.
           UpperCAmelCase      	=		len(self.content.keys()    )
           if num_first_level_keys > 1:
               # If more than one, add to the error list, continue
               error_list.append(
                   f"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()    )]    )}. Only one heading is expected. Skipping further validation for this README."    )
           elif num_first_level_keys < 1:
               # If less than one, append error.
               error_list.append(
                   '''The README has no first-level headings. One heading is expected. Skipping further validation for this README.'''    )

           else:
               # If one exactly
               UpperCAmelCase      	=		list(self.content.keys()    )[0]  # Get the key
               if start_key.startswith('''Dataset Card for'''    ):  # Check correct start
                   # If the starting is correct, validate all the sections
                   UpperCAmelCase      	=		self.content[start_key].validate(
                       readme_structure['''subsections'''][0]    )
                   error_list += sec_error_list
                   warning_list += sec_warning_list
               else:
                   # If not found, append error
                   error_list.append(
                       '''No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'''    )
           if error_list:
               # If there are errors, do not return the dictionary as it is invalid
               return {}, error_list, warning_list
           else:
               return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
   from argparse import ArgumentParser

   A			   =ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
   ap.add_argument('readme_filepath')
   A			   =ap.parse_args()
   A			   =Path(args.readme_filepath)
   A			   =ReadMe.from_readme(readme_filepath)











'''simple docstring'''







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					Dict           = RealmTokenizer
       __a    :					Dict           = RealmTokenizerFast
       __a    :					Union[str, Any]           = True
       __a    :					Optional[int]           = True
       __a    :					int           = filter_non_english
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           super().setUp()

           UpperCAmelCase      	=		[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     int       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''
           UpperCAmelCase      	=		'''unwanted, running'''
           return input_text, output_text
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''UNwant\u00E9d,running'''    )
           self.assertListEqual(lowercase       ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [9, 6, 7, 12, 10, 11]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase      	=		self.get_tokenizer()
           UpperCAmelCase      	=		self.get_rust_tokenizer()

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           # With lower casing
           UpperCAmelCase      	=		self.get_tokenizer(do_lower_case=lowercase    )
           UpperCAmelCase      	=		self.get_rust_tokenizer(do_lower_case=lowercase    )

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )       ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''h\u00E9llo''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    never_split=['''[UNK]''']    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           UpperCAmelCase      	=		{}
           for i, token in enumerate(lowercase    ):
               UpperCAmelCase      	=		i
           UpperCAmelCase      	=		WordpieceTokenizer(vocab=lowercase       ,    unk_token='''[UNK]'''    )

           self.assertListEqual(tokenizer.tokenize(''''''    )       ,    []    )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )       ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )       ,    ['''[UNK]''', '''runn''', '''##ing''']    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''    )    )
           self.assertTrue(_is_whitespace('''\t'''    )    )
           self.assertTrue(_is_whitespace('''\r'''    )    )
           self.assertTrue(_is_whitespace('''\n'''    )    )
           self.assertTrue(_is_whitespace('''\u00A0'''    )    )

           self.assertFalse(_is_whitespace('''A'''    )    )
           self.assertFalse(_is_whitespace('''-'''    )    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''    )    )

           self.assertFalse(_is_control('''A'''    )    )
           self.assertFalse(_is_control(''' '''    )    )
           self.assertFalse(_is_control('''\t'''    )    )
           self.assertFalse(_is_control('''\r'''    )    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''    )    )
           self.assertTrue(_is_punctuation('''$'''    )    )
           self.assertTrue(_is_punctuation('''`'''    )    )
           self.assertTrue(_is_punctuation('''.'''    )    )

           self.assertFalse(_is_punctuation('''A'''    )    )
           self.assertFalse(_is_punctuation(''' '''    )    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )

           if self.test_rust_tokenizer:
               UpperCAmelCase      	=		self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )
       @slow
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''    )

           UpperCAmelCase      	=		tokenizer.encode('''sequence builders'''       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		tokenizer.encode('''multi-sequence build'''       ,    add_special_tokens=lowercase    )

           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase    )
           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase       ,    lowercase    )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]
       def    A				(     self						:     int    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )

                   UpperCAmelCase      	=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                   UpperCAmelCase      	=		tokenizer_r.encode_plus(
                       lowercase       ,    return_attention_mask=lowercase       ,    return_token_type_ids=lowercase       ,    return_offsets_mapping=lowercase       ,    add_special_tokens=lowercase       ,    )

                   UpperCAmelCase      	=		tokenizer_r.do_lower_case if hasattr(lowercase       ,    '''do_lower_case'''    ) else False
                   UpperCAmelCase      	=		(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
                   self.assertEqual([e[0] for e in expected_results]       ,    tokens['''offset_mapping''']    )







       @slow
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		self.tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   UpperCAmelCase      	=		tokenizer_r.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )
                   UpperCAmelCase      	=		tokenizer_p.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )

                   UpperCAmelCase      	=		(2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape       ,    lowercase    )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape       ,    lowercase    )




