
"""simple docstring"""
import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_url
from PIL import Image

from transformers import DeformableDetrConfig, DeformableDetrForObjectDetection, DeformableDetrImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_			      =					logging.get_logger(__name__)

def 				lowercase      (_lowerCAmelCase		):
						if "backbone.0.body" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""backbone.0.body"""				,  """backbone.conv_encoder.model"""		)
						if "transformer" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""transformer."""				,  """"""		)
						if "norm1" in orig_key:
												if "encoder" in orig_key:
																		__lowerCAmelCase        =       orig_key.replace("""norm1"""				,  """self_attn_layer_norm"""		)
												else:
																		__lowerCAmelCase        =       orig_key.replace("""norm1"""				,  """encoder_attn_layer_norm"""		)
						if "norm2" in orig_key:
												if "encoder" in orig_key:
																		__lowerCAmelCase        =       orig_key.replace("""norm2"""				,  """final_layer_norm"""		)
												else:
																		__lowerCAmelCase        =       orig_key.replace("""norm2"""				,  """self_attn_layer_norm"""		)
						if "norm3" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""norm3"""				,  """final_layer_norm"""		)
						if "linear1" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""linear1"""				,  """fc1"""		)
						if "linear2" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""linear2"""				,  """fc2"""		)
						if "query_embed" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""query_embed"""				,  """query_position_embeddings"""		)
						if "cross_attn" in orig_key:
												__lowerCAmelCase        =       orig_key.replace("""cross_attn"""				,  """encoder_attn"""		)

						return orig_key

def 				lowercase      (_lowerCAmelCase		):
						# transformer decoder self-attention layers
						for i in range(6		):
												# read in weights + bias of input projection layer of self-attention
												__lowerCAmelCase        =       state_dict.pop(f"""decoder.layers.{i}.self_attn.in_proj_weight"""		)
												__lowerCAmelCase        =       state_dict.pop(f"""decoder.layers.{i}.self_attn.in_proj_bias"""		)
												# next, add query, keys and values (in that order) to the state dict
												__lowerCAmelCase        =       in_proj_weight[:256, :]
												__lowerCAmelCase        =       in_proj_bias[:256]
												__lowerCAmelCase        =       in_proj_weight[256:512, :]
												__lowerCAmelCase        =       in_proj_bias[256:512]
												__lowerCAmelCase        =       in_proj_weight[-256:, :]
												__lowerCAmelCase        =       in_proj_bias[-256:]

def 				lowercase      ():
						__lowerCAmelCase        =       "http://images.cocodataset.org/val2017/000000039769.jpg"
						__lowerCAmelCase        =       Image.open(requests.get(_SCREAMING_SNAKE_CASE				,  stream=_SCREAMING_SNAKE_CASE		).raw		)

						return im

@torch.no_grad()
def 				lowercase      (_lowerCAmelCase				,  _lowerCAmelCase				,  _lowerCAmelCase				,  _lowerCAmelCase				,  _lowerCAmelCase				,  _lowerCAmelCase				,  _lowerCAmelCase				,  ):
						__lowerCAmelCase        =       DeformableDetrConfig()
						# set config attributes
						if single_scale:
												__lowerCAmelCase        =       1
						__lowerCAmelCase        =       dilation
						__lowerCAmelCase        =       with_box_refine
						__lowerCAmelCase        =       two_stage
						# set labels
						__lowerCAmelCase        =       91
						__lowerCAmelCase        =       "huggingface/label-files"
						__lowerCAmelCase        =       "coco-detection-id2label.json"
						__lowerCAmelCase        =       json.load(open(cached_download(hf_hub_url(_SCREAMING_SNAKE_CASE				,  _SCREAMING_SNAKE_CASE				,  repo_type="""dataset"""		)		)				,  """r"""		)		)
						__lowerCAmelCase        =       {int(_SCREAMING_SNAKE_CASE		): v for k, v in idalabel.items()}
						__lowerCAmelCase        =       idalabel
						__lowerCAmelCase        =       {v: k for k, v in idalabel.items()}

						# load image processor
						__lowerCAmelCase        =       DeformableDetrImageProcessor(format="""coco_detection"""		)

						# prepare image
						__lowerCAmelCase        =       prepare_img()
						__lowerCAmelCase        =       image_processor(images=_SCREAMING_SNAKE_CASE				,  return_tensors="""pt"""		)
						__lowerCAmelCase        =       encoding["pixel_values"]

						logger.info("""Converting model..."""		)

						# load original state dict
						__lowerCAmelCase        =       torch.load(_SCREAMING_SNAKE_CASE				,  map_location="""cpu"""		)["model"]
						# rename keys
						for key in state_dict.copy().keys():
												__lowerCAmelCase        =       state_dict.pop(_SCREAMING_SNAKE_CASE		)
												__lowerCAmelCase        =       val
						# query, key and value matrices need special treatment
						read_in_q_k_v(_SCREAMING_SNAKE_CASE		)
						# important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
						__lowerCAmelCase        =       "model."
						for key in state_dict.copy().keys():
												if not key.startswith("""class_embed"""		) and not key.startswith("""bbox_embed"""		):
																		__lowerCAmelCase        =       state_dict.pop(_SCREAMING_SNAKE_CASE		)
																		__lowerCAmelCase        =       val
    # finally, create HuggingFace model and load state dict
						__lowerCAmelCase        =       DeformableDetrForObjectDetection(_SCREAMING_SNAKE_CASE		)
						model.load_state_dict(_SCREAMING_SNAKE_CASE		)
						model.eval()

						__lowerCAmelCase        =       "cuda" if torch.cuda.is_available() else "cpu"
						model.to(_SCREAMING_SNAKE_CASE		)
						# verify our conversion
						__lowerCAmelCase        =       model(pixel_values.to(_SCREAMING_SNAKE_CASE		)		)

						__lowerCAmelCase        =       torch.tensor(
						    [[-9.6_645, -4.3_449, -5.8_705], [-9.7_035, -3.8_504, -5.0_724], [-10.5_634, -5.3_379, -7.5_116]]		)
						__lowerCAmelCase        =       torch.tensor([[0.8_693, 0.2_289, 0.2_492], [0.3_150, 0.5_489, 0.5_845], [0.5_563, 0.7_580, 0.8_518]]		)

						if single_scale:
												__lowerCAmelCase        =       torch.tensor(
												    [[-9.9_051, -4.2_541, -6.4_852], [-9.6_947, -4.0_854, -6.8_033], [-10.0_665, -5.8_470, -7.7_003]]		)
												__lowerCAmelCase        =       torch.tensor([[0.7_292, 0.4_991, 0.5_532], [0.7_959, 0.2_426, 0.4_236], [0.7_582, 0.3_518, 0.4_451]]		)

						if single_scale and dilation:
												__lowerCAmelCase        =       torch.tensor(
												    [[-8.9_652, -4.1_074, -5.6_635], [-9.0_596, -4.9_447, -6.6_075], [-10.1_178, -4.5_275, -6.2_671]]		)
												__lowerCAmelCase        =       torch.tensor([[0.7_665, 0.4_130, 0.4_769], [0.8_364, 0.1_841, 0.3_391], [0.6_261, 0.3_895, 0.7_978]]		)

						if with_box_refine:
												__lowerCAmelCase        =       torch.tensor(
												    [[-8.8_895, -5.4_187, -6.8_153], [-8.4_706, -6.1_668, -7.6_184], [-9.0_042, -5.5_359, -6.9_141]]		)
												__lowerCAmelCase        =       torch.tensor([[0.7_828, 0.2_208, 0.4_323], [0.0_892, 0.5_996, 0.1_319], [0.5_524, 0.6_389, 0.8_914]]		)

						if with_box_refine and two_stage:
												__lowerCAmelCase        =       torch.tensor(
												    [[-6.7_108, -4.3_213, -6.3_777], [-8.9_014, -6.1_799, -6.7_240], [-6.9_315, -4.4_735, -6.2_298]]		)
												__lowerCAmelCase        =       torch.tensor([[0.2_583, 0.5_499, 0.4_683], [0.7_652, 0.9_068, 0.4_882], [0.5_490, 0.2_763, 0.0_564]]		)

						print("""Logits:"""				,  outputs.logits[0, :3, :3]		)

						assert torch.allclose(outputs.logits[0, :3, :3]				,  expected_logits.to(_SCREAMING_SNAKE_CASE		)				,  atol=1E-4		)
						assert torch.allclose(outputs.pred_boxes[0, :3, :3]				,  expected_boxes.to(_SCREAMING_SNAKE_CASE		)				,  atol=1E-4		)

						print("""Everything ok!"""		)

						# Save model and image processor
						logger.info(f"""Saving PyTorch model and image processor to {pytorch_dump_folder_path}..."""		)
						Path(_SCREAMING_SNAKE_CASE		).mkdir(exist_ok=_SCREAMING_SNAKE_CASE		)
						model.save_pretrained(_SCREAMING_SNAKE_CASE		)
						image_processor.save_pretrained(_SCREAMING_SNAKE_CASE		)

						# Push to hub
						if push_to_hub:
												__lowerCAmelCase        =       "deformable-detr"
												model_name += "-single-scale" if single_scale else ""
												model_name += "-dc5" if dilation else ""
												model_name += "-with-box-refine" if with_box_refine else ""
												model_name += "-two-stage" if two_stage else ""
												print("""Pushing model to hub..."""		)
												model.push_to_hub(repo_path_or_name=_SCREAMING_SNAKE_CASE				,  organization="""nielsr"""				,  commit_message="""Add model"""		)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_			      =					argparse.ArgumentParser()

					parser.add_argument(
					    '''--checkpoint_path''',
					    type=str,
					    default='''/home/niels/checkpoints/deformable_detr/r50_deformable_detr-checkpoint.pth''',
					    help='''Path to Pytorch checkpoint (.pth file) you\'d like to convert.''',
					)
					parser.add_argument('''--single_scale''', action='''store_true''', help='''Whether to set config.num_features_levels = 1.''')
					parser.add_argument('''--dilation''', action='''store_true''', help='''Whether to set config.dilation=True.''')
					parser.add_argument('''--with_box_refine''', action='''store_true''', help='''Whether to set config.with_box_refine=True.''')
					parser.add_argument('''--two_stage''', action='''store_true''', help='''Whether to set config.two_stage=True.''')
					parser.add_argument(
					    '''--pytorch_dump_folder_path''',
					    default=None,
					    type=str,
					    required=True,
					    help='''Path to the folder to output PyTorch model.''',
					)
					parser.add_argument(
					    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
					)
					SCREAMING_SNAKE_CASE_			      =					parser.parse_args()
					convert_deformable_detr_checkpoint(
					    args.checkpoint_path,
					    args.single_scale,
					    args.dilation,
					    args.with_box_refine,
					    args.two_stage,
					    args.pytorch_dump_folder_path,
					    args.push_to_hub,
					)


"""simple docstring"""
import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowerCAmelCase_						(							A__							,							unittest.TestCase				):



							'''simple docstring'''





							_snake_case		     =						ElectraTokenizer
							_snake_case		     =						ElectraTokenizerFast
							_snake_case		     =						True
							_snake_case		     =						True
							_snake_case		     =						filter_non_english




							def  A__		(				self							)   ->       Union[str, Any]:
													super().setUp()

													__lowerCAmelCase        =       [
													    """[UNK]""",
													    """[CLS]""",
													    """[SEP]""",
													    """[PAD]""",
													    """[MASK]""",
													    """want""",
													    """##want""",
													    """##ed""",
													    """wa""",
													    """un""",
													    """runn""",
													    """##ing""",
													    """,""",
													    """low""",
													    """lowest""",
													]
													__lowerCAmelCase        =       os.path.join(self.tmpdirname  ,							VOCAB_FILES_NAMES["""vocab_file"""]							)
													with open(self.vocab_file  ,							"""w"""  ,							encoding="""utf-8"""							) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)




							def  A__		(				self  ,							snake_case_							)   ->       str:
													__lowerCAmelCase        =       """UNwant\u00E9d,running"""
													__lowerCAmelCase        =       """unwanted, running"""
													return input_text, output_text




							def  A__		(				self							)   ->       List[Any]:
													__lowerCAmelCase        =       self.tokenizer_class(self.vocab_file							)

													__lowerCAmelCase        =       tokenizer.tokenize("""UNwant\u00E9d,running"""							)
													self.assertListEqual(snake_case_  ,							["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_							)  ,							[9, 6, 7, 12, 10, 11]							)




							def  A__		(				self							)   ->       Any:
													if not self.test_rust_tokenizer:
																			return

													__lowerCAmelCase        =       self.get_tokenizer()
													__lowerCAmelCase        =       self.get_rust_tokenizer()

													__lowerCAmelCase        =       """UNwant\u00E9d,running"""

													__lowerCAmelCase        =       tokenizer.tokenize(snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.tokenize(snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)

													__lowerCAmelCase        =       tokenizer.encode(snake_case_  ,							add_special_tokens=snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.encode(snake_case_  ,							add_special_tokens=snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)

													__lowerCAmelCase        =       self.get_rust_tokenizer()
													__lowerCAmelCase        =       tokenizer.encode(snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.encode(snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)

													# With lower casing
													__lowerCAmelCase        =       self.get_tokenizer(do_lower_case=snake_case_							)
													__lowerCAmelCase        =       self.get_rust_tokenizer(do_lower_case=snake_case_							)

													__lowerCAmelCase        =       """UNwant\u00E9d,running"""

													__lowerCAmelCase        =       tokenizer.tokenize(snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.tokenize(snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)

													__lowerCAmelCase        =       tokenizer.encode(snake_case_  ,							add_special_tokens=snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.encode(snake_case_  ,							add_special_tokens=snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)

													__lowerCAmelCase        =       self.get_rust_tokenizer()
													__lowerCAmelCase        =       tokenizer.encode(snake_case_							)
													__lowerCAmelCase        =       rust_tokenizer.encode(snake_case_							)
													self.assertListEqual(snake_case_  ,							snake_case_							)




							def  A__		(				self							)   ->       Union[str, Any]:
													__lowerCAmelCase        =       BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""							)  ,							["""ah""", """\u535A""", """\u63A8""", """zz"""]							)




							def  A__		(				self							)   ->       str:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)  ,							["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
													self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)  ,							["""hello"""]							)




							def  A__		(				self							)   ->       str:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_  ,							strip_accents=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)  ,							["""hällo""", """!""", """how""", """are""", """you""", """?"""]							)
													self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)  ,							["""h\u00E9llo"""]							)




							def  A__		(				self							)   ->       int:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_  ,							strip_accents=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)  ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
													self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)  ,							["""hello"""]							)




							def  A__		(				self							)   ->       Dict:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)  ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
													self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)  ,							["""hello"""]							)




							def  A__		(				self							)   ->       List[str]:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)  ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)




							def  A__		(				self							)   ->       Any:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_  ,							strip_accents=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)  ,							["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)




							def  A__		(				self							)   ->       Tuple:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_  ,							strip_accents=snake_case_							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)  ,							["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)




							def  A__		(				self							)   ->       Dict:
													__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_  ,							never_split=["""[UNK]"""]							)

													self.assertListEqual(
													    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""							)  ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]							)




							def  A__		(				self							)   ->       List[str]:
													__lowerCAmelCase        =       ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

													__lowerCAmelCase        =       {}
													for i, token in enumerate(snake_case_							):
																			__lowerCAmelCase        =       i
													__lowerCAmelCase        =       WordpieceTokenizer(vocab=snake_case_  ,							unk_token="""[UNK]"""							)

													self.assertListEqual(tokenizer.tokenize(""""""							)  ,							[]							)

													self.assertListEqual(tokenizer.tokenize("""unwanted running"""							)  ,							["""un""", """##want""", """##ed""", """runn""", """##ing"""]							)

													self.assertListEqual(tokenizer.tokenize("""unwantedX running"""							)  ,							["""[UNK]""", """runn""", """##ing"""]							)




							def  A__		(				self							)   ->       str:
													self.assertTrue(_is_whitespace(""" """							)							)
													self.assertTrue(_is_whitespace("""\t"""							)							)
													self.assertTrue(_is_whitespace("""\r"""							)							)
													self.assertTrue(_is_whitespace("""\n"""							)							)
													self.assertTrue(_is_whitespace("""\u00A0"""							)							)

													self.assertFalse(_is_whitespace("""A"""							)							)
													self.assertFalse(_is_whitespace("""-"""							)							)




							def  A__		(				self							)   ->       Optional[int]:
													self.assertTrue(_is_control("""\u0005"""							)							)

													self.assertFalse(_is_control("""A"""							)							)
													self.assertFalse(_is_control(""" """							)							)
													self.assertFalse(_is_control("""\t"""							)							)
													self.assertFalse(_is_control("""\r"""							)							)




							def  A__		(				self							)   ->       int:
													self.assertTrue(_is_punctuation("""-"""							)							)
													self.assertTrue(_is_punctuation("""$"""							)							)
													self.assertTrue(_is_punctuation("""`"""							)							)
													self.assertTrue(_is_punctuation("""."""							)							)

													self.assertFalse(_is_punctuation("""A"""							)							)
													self.assertFalse(_is_punctuation(""" """							)							)




							def  A__		(				self							)   ->       str:
													__lowerCAmelCase        =       self.get_tokenizer()
													__lowerCAmelCase        =       self.get_rust_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(snake_case_							) for t in ["""Test""", """\xad""", """test"""]]  ,							[["""[UNK]"""], [], ["""[UNK]"""]]							)

													self.assertListEqual(
													    [rust_tokenizer.tokenize(snake_case_							) for t in ["""Test""", """\xad""", """test"""]]  ,							[["""[UNK]"""], [], ["""[UNK]"""]]							)




							@slow
							def  A__		(				self							)   ->       List[str]:
													__lowerCAmelCase        =       self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""							)

													__lowerCAmelCase        =       tokenizer.encode("""sequence builders"""  ,							add_special_tokens=snake_case_							)
													__lowerCAmelCase        =       tokenizer.encode("""multi-sequence build"""  ,							add_special_tokens=snake_case_							)

													__lowerCAmelCase        =       tokenizer.build_inputs_with_special_tokens(snake_case_							)
													__lowerCAmelCase        =       tokenizer.build_inputs_with_special_tokens(snake_case_  ,							snake_case_							)

													assert encoded_sentence == [101] + text + [102]
													assert encoded_pair == [101] + text + [102] + text_a + [102]




							def  A__		(				self							)   ->       int:
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																									__lowerCAmelCase        =       self.rust_tokenizer_class.from_pretrained(snake_case_  ,							**snake_case_							)

																									__lowerCAmelCase        =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																									__lowerCAmelCase        =       tokenizer_r.encode_plus(
																									    snake_case_  ,							return_attention_mask=snake_case_  ,							return_token_type_ids=snake_case_  ,							return_offsets_mapping=snake_case_  ,							add_special_tokens=snake_case_  ,							)

																									__lowerCAmelCase        =       tokenizer_r.do_lower_case if hasattr(snake_case_  ,							"""do_lower_case"""							) else False
																									__lowerCAmelCase        =       (
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), """A"""),
																									        ((1, 2), ""","""),
																									        ((3, 5), """na"""),
																									        ((5, 6), """##ï"""),
																									        ((6, 8), """##ve"""),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), """Allen"""),
																									        ((21, 23), """##NL"""),
																									        ((23, 24), """##P"""),
																									        ((25, 33), """sentence"""),
																									        ((33, 34), """."""),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), """a"""),
																									        ((1, 2), ""","""),
																									        ((3, 8), """naive"""),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), """allen"""),
																									        ((21, 23), """##nl"""),
																									        ((23, 24), """##p"""),
																									        ((25, 33), """sentence"""),
																									        ((33, 34), """."""),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]  ,							tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]							)							)
																									self.assertEqual([e[0] for e in expected_results]  ,							tokens["""offset_mapping"""]							)




							def  A__		(				self							)   ->       List[Any]:
													__lowerCAmelCase        =       ["""的""", """人""", """有"""]
													__lowerCAmelCase        =       """""".join(snake_case_							)
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																									__lowerCAmelCase        =       True
																									__lowerCAmelCase        =       self.tokenizer_class.from_pretrained(snake_case_  ,							**snake_case_							)
																									__lowerCAmelCase        =       self.rust_tokenizer_class.from_pretrained(snake_case_  ,							**snake_case_							)

																									__lowerCAmelCase        =       tokenizer_p.encode(snake_case_  ,							add_special_tokens=snake_case_							)
																									__lowerCAmelCase        =       tokenizer_r.encode(snake_case_  ,							add_special_tokens=snake_case_							)

																									__lowerCAmelCase        =       tokenizer_r.convert_ids_to_tokens(snake_case_							)
																									__lowerCAmelCase        =       tokenizer_p.convert_ids_to_tokens(snake_case_							)

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(snake_case_  ,							snake_case_							)
																									self.assertListEqual(snake_case_  ,							snake_case_							)

																									__lowerCAmelCase        =       False
																									__lowerCAmelCase        =       self.rust_tokenizer_class.from_pretrained(snake_case_  ,							**snake_case_							)
																									__lowerCAmelCase        =       self.tokenizer_class.from_pretrained(snake_case_  ,							**snake_case_							)

																									__lowerCAmelCase        =       tokenizer_r.encode(snake_case_  ,							add_special_tokens=snake_case_							)
																									__lowerCAmelCase        =       tokenizer_p.encode(snake_case_  ,							add_special_tokens=snake_case_							)

																									__lowerCAmelCase        =       tokenizer_r.convert_ids_to_tokens(snake_case_							)
																									__lowerCAmelCase        =       tokenizer_p.convert_ids_to_tokens(snake_case_							)

																									# it is expected that only the first Chinese character is not preceded by "##".
																									__lowerCAmelCase        =       [
																									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case_							)
																									]
																									self.assertListEqual(snake_case_  ,							snake_case_							)
																									self.assertListEqual(snake_case_  ,							snake_case_							)
