




"""simple docstring"""

import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
_lowerCamelCase					:     str				       =    logging.get_logger(__name__)

def 						lowercase_ (						):




	"""simple docstring"""






	A_					:			Union[str, Any]						=						DetaConfig(
	    num_queries=900					, encoder_ffn_dim=2048					, decoder_ffn_dim=2048					, num_feature_levels=5					, assign_first_stage=_UpperCAmelCase					, with_box_refine=_UpperCAmelCase					, two_stage=_UpperCAmelCase					, )

	# set labels
	A_					:			Union[str, Any]						=						91
	A_					:			Any						=						'''huggingface/label-files'''
	A_					:			Dict						=						'''coco-detection-id2label.json'''
	A_					:			List[str]						=						json.load(open(cached_download(hf_hub_url(_UpperCAmelCase					, _UpperCAmelCase					, repo_type='''dataset'''		)		)					, '''r'''		)		)
	A_					:			Optional[int]						=						{int(_UpperCAmelCase		): v for k, v in idalabel.items()}
	A_					:			List[str]						=						idalabel
	A_					:			Tuple						=						{v: k for k, v in idalabel.items()}

	return config

def 						lowercase_ (						_UpperCAmelCase		):




	"""simple docstring"""






	A_					:			int						=						[]

	# stem
	# fmt: off
	rename_keys.append(('''backbone.0.body.conv1.weight''', '''model.backbone.model.embedder.embedder.convolution.weight''')		)
	rename_keys.append(('''backbone.0.body.bn1.weight''', '''model.backbone.model.embedder.embedder.normalization.weight''')		)
	rename_keys.append(('''backbone.0.body.bn1.bias''', '''model.backbone.model.embedder.embedder.normalization.bias''')		)
	rename_keys.append(('''backbone.0.body.bn1.running_mean''', '''model.backbone.model.embedder.embedder.normalization.running_mean''')		)
	rename_keys.append(('''backbone.0.body.bn1.running_var''', '''model.backbone.model.embedder.embedder.normalization.running_var''')		)
	# stages
	for stage_idx in range(len(config.backbone_config.depths		)		):
		for layer_idx in range(config.backbone_config.depths[stage_idx]		):
			# shortcut
			if layer_idx == 0:
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var""",
				    )		)
			# 3 convs
			for i in range(3		):
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean""",
				    )		)
				rename_keys.append(
				    (
				        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var""",
				        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var""",
				    )		)
    # transformer encoder
	for i in range(config.encoder_layers		):
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight""", f"""model.encoder.layers.{i}.self_attn.sampling_offsets.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias""", f"""model.encoder.layers.{i}.self_attn.sampling_offsets.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.attention_weights.weight""", f"""model.encoder.layers.{i}.self_attn.attention_weights.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.attention_weights.bias""", f"""model.encoder.layers.{i}.self_attn.attention_weights.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.value_proj.weight""", f"""model.encoder.layers.{i}.self_attn.value_proj.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.value_proj.bias""", f"""model.encoder.layers.{i}.self_attn.value_proj.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.output_proj.weight""", f"""model.encoder.layers.{i}.self_attn.output_proj.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.output_proj.bias""", f"""model.encoder.layers.{i}.self_attn.output_proj.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.norm1.weight""", f"""model.encoder.layers.{i}.self_attn_layer_norm.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.norm1.bias""", f"""model.encoder.layers.{i}.self_attn_layer_norm.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.weight""", f"""model.encoder.layers.{i}.fc1.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.bias""", f"""model.encoder.layers.{i}.fc1.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.weight""", f"""model.encoder.layers.{i}.fc2.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.bias""", f"""model.encoder.layers.{i}.fc2.bias""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.weight""", f"""model.encoder.layers.{i}.final_layer_norm.weight""")		)
		rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.bias""", f"""model.encoder.layers.{i}.final_layer_norm.bias""")		)

	# transformer decoder
	for i in range(config.decoder_layers		):
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight""", f"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias""", f"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.attention_weights.weight""", f"""model.decoder.layers.{i}.encoder_attn.attention_weights.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.attention_weights.bias""", f"""model.decoder.layers.{i}.encoder_attn.attention_weights.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.value_proj.weight""", f"""model.decoder.layers.{i}.encoder_attn.value_proj.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.value_proj.bias""", f"""model.decoder.layers.{i}.encoder_attn.value_proj.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.output_proj.weight""", f"""model.decoder.layers.{i}.encoder_attn.output_proj.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.output_proj.bias""", f"""model.decoder.layers.{i}.encoder_attn.output_proj.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm1.weight""", f"""model.decoder.layers.{i}.encoder_attn_layer_norm.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm1.bias""", f"""model.decoder.layers.{i}.encoder_attn_layer_norm.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.self_attn.out_proj.weight""", f"""model.decoder.layers.{i}.self_attn.out_proj.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.self_attn.out_proj.bias""", f"""model.decoder.layers.{i}.self_attn.out_proj.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm2.weight""", f"""model.decoder.layers.{i}.self_attn_layer_norm.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm2.bias""", f"""model.decoder.layers.{i}.self_attn_layer_norm.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.weight""", f"""model.decoder.layers.{i}.fc1.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.bias""", f"""model.decoder.layers.{i}.fc1.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.weight""", f"""model.decoder.layers.{i}.fc2.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.bias""", f"""model.decoder.layers.{i}.fc2.bias""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.weight""", f"""model.decoder.layers.{i}.final_layer_norm.weight""")		)
		rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.bias""", f"""model.decoder.layers.{i}.final_layer_norm.bias""")		)

	# fmt: on

	return rename_keys

def 						lowercase_ (						_UpperCAmelCase					, _UpperCAmelCase					, _UpperCAmelCase		):




	"""simple docstring"""






	A_					:			List[Any]						=						dct.pop(_UpperCAmelCase		)
	A_					:			Any						=						val

def 						lowercase_ (						_UpperCAmelCase					, _UpperCAmelCase		):




	"""simple docstring"""






	A_					:			Optional[Any]						=						config.d_model
	for i in range(config.decoder_layers		):
		# read in weights + bias of input projection layer of self-attention
		A_					:			Dict						=						state_dict.pop(f"""transformer.decoder.layers.{i}.self_attn.in_proj_weight"""		)
		A_					:			Any						=						state_dict.pop(f"""transformer.decoder.layers.{i}.self_attn.in_proj_bias"""		)
		# next, add query, keys and values (in that order) to the state dict
		A_					:			Any						=						in_proj_weight[:hidden_size, :]
		A_					:			List[str]						=						in_proj_bias[:hidden_size]
		A_					:			Optional[Any]						=						in_proj_weight[
		    hidden_size : hidden_size * 2, :
		]
		A_					:			List[Any]						=						in_proj_bias[hidden_size : hidden_size * 2]
		A_					:			Optional[int]						=						in_proj_weight[-hidden_size:, :]
		A_					:			Optional[int]						=						in_proj_bias[-hidden_size:]

def 						lowercase_ (						):




	"""simple docstring"""






	A_					:			Any						=						'''http://images.cocodataset.org/val2017/000000039769.jpg'''
	A_					:			int						=						Image.open(requests.get(_UpperCAmelCase					, stream=_UpperCAmelCase		).raw		)

	return im

@torch.no_grad()
def 						lowercase_ (						_UpperCAmelCase					, _UpperCAmelCase					, _UpperCAmelCase		):




	"""simple docstring"""






	A_					:			str						=						get_deta_config()

	# load original state dict
	if model_name == "deta-resnet-50":
		A_					:			str						=						'''adet_checkpoint0011.pth'''
	elif model_name == "deta-resnet-50-24-epochs":
		A_					:			List[Any]						=						'''adet_2x_checkpoint0023.pth'''
	else:
		raise ValueError(f"""Model name {model_name} not supported"""		)
	A_					:			str						=						hf_hub_download(repo_id='''nielsr/deta-checkpoints'''					, filename=_UpperCAmelCase		)
	A_					:			Any						=						torch.load(_UpperCAmelCase					, map_location='''cpu'''		)['''model''']

	# rename keys
	A_					:			Optional[Any]						=						create_rename_keys(_UpperCAmelCase		)
	for src, dest in rename_keys:
		rename_key(_UpperCAmelCase					, _UpperCAmelCase					, _UpperCAmelCase		)
	read_in_decoder_q_k_v(_UpperCAmelCase					, _UpperCAmelCase		)

	# fix some prefixes
	for key in state_dict.copy().keys():
		if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
			A_					:			Dict						=						state_dict.pop(_UpperCAmelCase		)
			A_					:			int						=						val
		if "input_proj" in key:
			A_					:			Dict						=						state_dict.pop(_UpperCAmelCase		)
			A_					:			str						=						val
		if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
			A_					:			int						=						state_dict.pop(_UpperCAmelCase		)
			A_					:			Tuple						=						val

    # finally, create HuggingFace model and load state dict
	A_					:			str						=						DetaForObjectDetection(_UpperCAmelCase		)
	model.load_state_dict(_UpperCAmelCase		)
	model.eval()

	A_					:			Optional[int]						=						'''cuda''' if torch.cuda.is_available() else '''cpu'''
	model.to(_UpperCAmelCase		)

	# load image processor
	A_					:			Any						=						DetaImageProcessor(format='''coco_detection'''		)

	# verify our conversion on image
	A_					:			List[str]						=						prepare_img()
	A_					:			List[Any]						=						processor(images=_UpperCAmelCase					, return_tensors='''pt'''		)
	A_					:			Optional[int]						=						encoding['''pixel_values''']
	A_					:			Tuple						=						model(pixel_values.to(_UpperCAmelCase		)		)

	# verify logits
	if model_name == "deta-resnet-50":
		A_					:			Optional[int]						=						torch.tensor(
		    [[-7.3978, -2.5406, -4.1668], [-8.2684, -3.9933, -3.8096], [-7.0515, -3.7973, -5.8516]]		)
		A_					:			Optional[int]						=						torch.tensor([[0.5043, 0.4973, 0.9998], [0.2542, 0.5489, 0.4748], [0.5490, 0.2765, 0.0570]]		)
	elif model_name == "deta-resnet-50-24-epochs":
		A_					:			Optional[Any]						=						torch.tensor(
		    [[-7.1688, -2.4857, -4.8669], [-7.8630, -3.8154, -4.2674], [-7.2730, -4.1865, -5.5323]]		)
		A_					:			List[str]						=						torch.tensor([[0.5021, 0.4971, 0.9994], [0.2546, 0.5486, 0.4731], [0.1686, 0.1986, 0.2142]]		)

	assert torch.allclose(outputs.logits[0, :3, :3]					, expected_logits.to(_UpperCAmelCase		)					, atol=1E-4		)
	assert torch.allclose(outputs.pred_boxes[0, :3, :3]					, expected_boxes.to(_UpperCAmelCase		)					, atol=1E-4		)
	print('''Everything ok!'''		)

	if pytorch_dump_folder_path:
		# Save model and processor
		logger.info(f"""Saving PyTorch model and processor to {pytorch_dump_folder_path}..."""		)
		Path(_UpperCAmelCase		).mkdir(exist_ok=_UpperCAmelCase		)
		model.save_pretrained(_UpperCAmelCase		)
		processor.save_pretrained(_UpperCAmelCase		)

	# Push to hub
	if push_to_hub:
		print('''Pushing model and processor to hub...'''		)
		model.push_to_hub(f"""jozhang97/{model_name}"""		)
		processor.push_to_hub(f"""jozhang97/{model_name}"""		)


if __name__ == "__main__":
		_lowerCamelCase					:     Dict				       =    argparse.ArgumentParser()

		parser.add_argument(
		    '--model_name',
		    type=str,
		    default='deta-resnet-50',
		    choices=['deta-resnet-50', 'deta-resnet-50-24-epochs'],
		    help='Name of the model you\'d like to convert.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path',
		    default=None,
		    type=str,
		    help='Path to the folder to output PyTorch model.',
		)
		parser.add_argument(
		    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
		)
		_lowerCamelCase					:     Union[str, Any]				       =    parser.parse_args()
		convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)











"""simple docstring"""

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     lowercase (      __UpperCAmelCase       ,		unittest.TestCase):
				__lowerCAmelCase      :      Any					    =				ElectraTokenizer
				__lowerCAmelCase      :      Optional[int]					    =				ElectraTokenizerFast
				__lowerCAmelCase      :      Any					    =				True
				__lowerCAmelCase      :      Tuple					    =				True
				__lowerCAmelCase      :      Optional[Any]					    =				filter_non_english







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					super().setUp()

					A_					:			int						=						[
					    '''[UNK]''',
					    '''[CLS]''',
					    '''[SEP]''',
					    '''[PAD]''',
					    '''[MASK]''',
					    '''want''',
					    '''##want''',
					    '''##ed''',
					    '''wa''',
					    '''un''',
					    '''runn''',
					    '''##ing''',
					    ''',''',
					    '''low''',
					    '''lowest''',
					]
					A_					:			str						=						os.path.join(self.tmpdirname ,       VOCAB_FILES_NAMES['''vocab_file''']    )
					with open(self.vocab_file ,       '''w''' ,       encoding='''utf-8'''    ) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						'''UNwant\u00E9d,running'''
					A_					:			Optional[int]						=						'''unwanted, running'''
					return input_text, output_text







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Tuple						=						self.tokenizer_class(self.vocab_file    )

					A_					:			Dict						=						tokenizer.tokenize('''UNwant\u00E9d,running'''    )
					self.assertListEqual(_lowerCamelCase ,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase    ) ,       [9, 6, 7, 12, 10, 11]    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					if not self.test_rust_tokenizer:
						return

					A_					:			Any						=						self.get_tokenizer()
					A_					:			Optional[Any]						=						self.get_rust_tokenizer()

					A_					:			Union[str, Any]						=						'''UNwant\u00E9d,running'''

					A_					:			Optional[int]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			int						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Any						=						self.get_rust_tokenizer()
					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			List[Any]						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					# With lower casing
					A_					:			List[str]						=						self.get_tokenizer(do_lower_case=_lowerCamelCase    )
					A_					:			Any						=						self.get_rust_tokenizer(do_lower_case=_lowerCamelCase    )

					A_					:			List[Any]						=						'''UNwant\u00E9d,running'''

					A_					:			List[str]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			List[str]						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[Any]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			List[Any]						=						self.get_rust_tokenizer()
					A_					:			List[Any]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			Dict						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Union[str, Any]						=						BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    ) ,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )







				def    a_					(							self				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''h\u00E9llo''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Tuple						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			int						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       never_split=['''[UNK]''']    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

					A_					:			Any						=						{}
					for i, token in enumerate(_lowerCamelCase    ):
						A_					:			int						=						i
					A_					:			str						=						WordpieceTokenizer(vocab=_lowerCamelCase ,       unk_token='''[UNK]'''    )

					self.assertListEqual(tokenizer.tokenize(''''''    ) ,       []    )

					self.assertListEqual(tokenizer.tokenize('''unwanted running'''    ) ,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

					self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    ) ,       ['''[UNK]''', '''runn''', '''##ing''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_whitespace(''' '''    )    )
					self.assertTrue(_is_whitespace('''\t'''    )    )
					self.assertTrue(_is_whitespace('''\r'''    )    )
					self.assertTrue(_is_whitespace('''\n'''    )    )
					self.assertTrue(_is_whitespace('''\u00A0'''    )    )

					self.assertFalse(_is_whitespace('''A'''    )    )
					self.assertFalse(_is_whitespace('''-'''    )    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					self.assertTrue(_is_control('''\u0005'''    )    )

					self.assertFalse(_is_control('''A'''    )    )
					self.assertFalse(_is_control(''' '''    )    )
					self.assertFalse(_is_control('''\t'''    )    )
					self.assertFalse(_is_control('''\r'''    )    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_punctuation('''-'''    )    )
					self.assertTrue(_is_punctuation('''$'''    )    )
					self.assertTrue(_is_punctuation('''`'''    )    )
					self.assertTrue(_is_punctuation('''.'''    )    )

					self.assertFalse(_is_punctuation('''A'''    )    )
					self.assertFalse(_is_punctuation(''' '''    )    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						self.get_tokenizer()
					A_					:			int						=						self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )







				@slow
				def    a_					(							self				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''    )

					A_					:			Tuple						=						tokenizer.encode('''sequence builders''' ,       add_special_tokens=_lowerCamelCase    )
					A_					:			List[Any]						=						tokenizer.encode('''multi-sequence build''' ,       add_special_tokens=_lowerCamelCase    )

					A_					:			List[Any]						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase    )
					A_					:			Any						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase ,       _lowerCamelCase    )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			Optional[Any]						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[int]						=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							A_					:			Optional[int]						=						tokenizer_r.encode_plus(
							    _lowerCamelCase ,       return_attention_mask=_lowerCamelCase ,       return_token_type_ids=_lowerCamelCase ,       return_offsets_mapping=_lowerCamelCase ,       add_special_tokens=_lowerCamelCase ,       )

							A_					:			Union[str, Any]						=						tokenizer_r.do_lower_case if hasattr(_lowerCamelCase ,       '''do_lower_case'''    ) else False
							A_					:			List[Any]						=						(
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''A'''),
							        ((1, 2), ''','''),
							        ((3, 5), '''na'''),
							        ((5, 6), '''##ï'''),
							        ((6, 8), '''##ve'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''Allen'''),
							        ((21, 23), '''##NL'''),
							        ((23, 24), '''##P'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''a'''),
							        ((1, 2), ''','''),
							        ((3, 8), '''naive'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''allen'''),
							        ((21, 23), '''##nl'''),
							        ((23, 24), '''##p'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results] ,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
							self.assertEqual([e[0] for e in expected_results] ,       tokens['''offset_mapping''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''的''', '''人''', '''有''']
					A_					:			Optional[int]						=						''''''.join(_lowerCamelCase    )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			List[Any]						=						True
							A_					:			Any						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			str						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			List[Any]						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			Any						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			Union[str, Any]						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

							A_					:			List[Any]						=						False
							A_					:			Any						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			Union[str, Any]						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[Any]						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			int						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			str						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that only the first Chinese character is not preceded by "##".
							A_					:			Dict						=						[
							    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase    )
							]
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )





