




"""simple docstring"""

import argparse
import logging
import os
import sys
import tempfile
from pathlib import Path

import lightning_base
import pytest
import pytorch_lightning as pl
import torch
from convert_pl_checkpoint_to_hf import convert_pl_to_hf
from distillation import distill_main
from finetune import SummarizationModule, main
from huggingface_hub import list_models
from parameterized import parameterized
from run_eval import generate_summaries_or_translations
from torch import nn

from transformers import AutoConfig, AutoModelForSeqaSeqLM
from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
from utils import label_smoothed_nll_loss, lmap, load_json


logging.basicConfig(level=logging.DEBUG)

_lowerCamelCase					:     List[str]				       =    logging.getLogger()
_lowerCamelCase					:     int				       =    torch.cuda.is_available()
_lowerCamelCase					:     List[str]				       =    {
    'max_tokens_per_batch': None,
    'supervise_forward': True,
    'normalize_hidden': True,
    'label_smoothing': 0.2,
    'eval_max_gen_length': None,
    'eval_beams': 1,
    'val_metric': 'loss',
    'save_top_k': 1,
    'adafactor': True,
    'early_stopping_patience': 2,
    'logger_name': 'default',
    'length_penalty': 0.5,
    'cache_dir': '',
    'task': 'summarization',
    'num_workers': 2,
    'alpha_hid': 0,
    'freeze_embeds': True,
    'enc_only': False,
    'tgt_suffix': '',
    'resume_from_checkpoint': None,
    'sortish_sampler': True,
    'student_decoder_layers': 1,
    'val_check_interval': 1.0,
    'output_dir': '',
    'fp16': False,  # TODO(SS): set this to CUDA_AVAILABLE if ci installs apex or start using native amp
    'no_teacher': False,
    'fp16_opt_level': 'O1',
    'gpus': 1 if CUDA_AVAILABLE else 0,
    'n_tpu_cores': 0,
    'max_grad_norm': 1.0,
    'do_train': True,
    'do_predict': True,
    'accumulate_grad_batches': 1,
    'server_ip': '',
    'server_port': '',
    'seed': 42,
    'model_name_or_path': 'sshleifer/bart-tiny-random',
    'config_name': '',
    'tokenizer_name': 'facebook/bart-large',
    'do_lower_case': False,
    'learning_rate': 0.3,
    'lr_scheduler': 'linear',
    'weight_decay': 0.0,
    'adam_epsilon': 1E-08,
    'warmup_steps': 0,
    'max_epochs': 1,
    'train_batch_size': 2,
    'eval_batch_size': 2,
    'max_source_length': 12,
    'max_target_length': 12,
    'val_max_target_length': 12,
    'test_max_target_length': 12,
    'fast_dev_run': False,
    'no_cache': False,
    'n_train': -1,
    'n_val': -1,
    'n_test': -1,
    'student_encoder_layers': 1,
    'freeze_encoder': False,
    'auto_scale_batch_size': False,
    'overwrite_output_dir': False,
    'student': None,
}

def 						lowercase_ (						_UpperCAmelCase					, _UpperCAmelCase		):




	"""simple docstring"""






	A_					:			Tuple						=						'''\n'''.join(_UpperCAmelCase		)
	Path(_UpperCAmelCase		).open('''w'''		).writelines(_UpperCAmelCase		)


_lowerCamelCase					:     Optional[int]				       =    [' Sam ate lunch today.', 'Sams lunch ingredients.']
_lowerCamelCase					:     List[Any]				       =    ['A very interesting story about what I ate for lunch.', 'Avocado, celery, turkey, coffee']
_lowerCamelCase					:     List[str]				       =    'patrickvonplaten/t5-tiny-random'
_lowerCamelCase					:     Tuple				       =    'sshleifer/t5-tinier-random'
_lowerCamelCase					:     int				       =    'sshleifer/bart-tiny-random'
_lowerCamelCase					:     Tuple				       =    'sshleifer/tiny-mbart'
_lowerCamelCase					:     Tuple				       =    'sshleifer/tiny-marian-en-de'
_lowerCamelCase					:     Union[str, Any]				       =    'stas/tiny-wmt19-en-de'


_lowerCamelCase					:     Dict				       =    logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks

def 						lowercase_ (						_UpperCAmelCase		):




	"""simple docstring"""






	for split in ["train", "val", "test"]:
		_dump_articles(os.path.join(_UpperCAmelCase					, f"""{split}.source"""		)					, _UpperCAmelCase		)
		_dump_articles(os.path.join(_UpperCAmelCase					, f"""{split}.target"""		)					, _UpperCAmelCase		)
	return tmp_dir

class     lowercase (      __UpperCAmelCase):







				@classmethod
				def    a_					(							cls				:   List[Any]    ):



					"""simple docstring"""

					logging.disable(logging.CRITICAL    )  # remove noisy download output from tracebacks
					return cls







				@slow
				@require_torch_gpu
				def    a_					(							self				:   Tuple    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						list_models()
					A_					:			Dict						=						'''sshleifer'''
					A_					:			List[str]						=						[x.modelId for x in model_list if x.modelId.startswith(_lowerCamelCase    )]
					A_					:			Union[str, Any]						=						['''sshleifer/blenderbot-3B''', '''sshleifer/blenderbot-90M''']
					A_					:			Dict						=						[]
					for m in model_ids:
						if m in allowed_to_be_broken:
							continue
						try:
							AutoConfig.from_pretrained(_lowerCamelCase    )
						except Exception:
							failures.append(_lowerCamelCase    )
					assert not failures, F"""The following models could not be loaded through AutoConfig: {failures}"""







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						{'''student_encoder_layers''': 2, '''student_decoder_layers''': 1, '''no_teacher''': True}
					self._test_distiller_cli(_lowerCamelCase    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			List[str]						=						{
					    '''student_encoder_layers''': 2,
					    '''student_decoder_layers''': 1,
					    '''max_epochs''': 4,
					    '''val_check_interval''': 0.25,
					    '''alpha_hid''': 2.0,
					    '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
					}
					A_					:			int						=						self._test_distiller_cli(_lowerCamelCase ,       check_contents=_lowerCamelCase    )

					A_					:			str						=						list(Path(model.output_dir    ).glob('''*.ckpt'''    )    )
					self.assertEqual(1 ,       len(_lowerCamelCase    )    )
					A_					:			List[str]						=						list(Path(model.output_dir    ).glob('''**/*.bin'''    )    )
					self.assertEqual(len(_lowerCamelCase    ) ,       2    )
					A_					:			Tuple						=						lmap(str.strip ,       Path(model.hparams.data_dir    ).joinpath('''test.source'''    ).open().readlines()    )
					A_					:			List[Any]						=						tempfile.mktemp()  # XXX: not being cleaned up
					generate_summaries_or_translations(_lowerCamelCase ,       _lowerCamelCase ,       str(model.output_dir / '''best_tfmr'''    )    )
					self.assertTrue(Path(_lowerCamelCase    ).exists()    )

					A_					:			Tuple						=						self.get_auto_remove_tmp_dir()
					convert_pl_to_hf(ckpts[0] ,       transformer_ckpts[0].parent ,       _lowerCamelCase    )
					assert os.path.exists(os.path.join(_lowerCamelCase ,       '''pytorch_model.bin'''    )    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						AutoModelForSeqaSeqLM.from_pretrained(_lowerCamelCase    )
					A_				,	A_					:			List[str]						=						model.dummy_inputs['''input_ids'''], model.dummy_inputs['''attention_mask''']
					A_					:			Optional[int]						=						torch.tensor([[0, 4, 8, 2], [0, 8, 2, 1]] ,       dtype=torch.long ,       device=model.device    )
					A_					:			Tuple						=						target_ids[:, :-1].contiguous()  # Why this line?
					A_					:			Optional[int]						=						target_ids[:, 1:].clone()  # why clone?
					A_					:			Optional[Any]						=						model(
					    _lowerCamelCase ,       attention_mask=_lowerCamelCase ,       decoder_input_ids=_lowerCamelCase ,       labels=_lowerCamelCase ,       use_cache=_lowerCamelCase    ).loss

					A_					:			Optional[int]						=						model(_lowerCamelCase ,       attention_mask=_lowerCamelCase ,       decoder_input_ids=_lowerCamelCase ,       use_cache=_lowerCamelCase    ).logits

					A_					:			int						=						nn.functional.log_softmax(_lowerCamelCase ,       dim=-1    )
					A_				,	A_					:			Optional[int]						=						label_smoothed_nll_loss(
					    _lowerCamelCase ,       _lowerCamelCase ,       0.1 ,       ignore_index=model.config.pad_token_id    )
					with self.assertRaises(_lowerCamelCase    ):
						# TODO: understand why this breaks
						self.assertEqual(_lowerCamelCase ,       _lowerCamelCase    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			List[Any]						=						{
					    '''student_encoder_layers''': 2,
					    '''student_decoder_layers''': 1,
					    '''num_train_epochs''': 4,
					    '''val_check_interval''': 0.25,
					    '''alpha_hid''': 2.0,
					    '''task''': '''translation''',
					    '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
					    '''tokenizer_name''': MBART_TINY,
					    '''teacher''': MBART_TINY,
					    '''src_lang''': '''en_XX''',
					    '''tgt_lang''': '''ro_RO''',
					}
					A_					:			str						=						self._test_distiller_cli(_lowerCamelCase ,       check_contents=_lowerCamelCase    )
					assert model.model.config.model_type == "mbart"

					A_					:			Dict						=						list(Path(model.output_dir    ).glob('''*.ckpt'''    )    )
					self.assertEqual(1 ,       len(_lowerCamelCase    )    )
					A_					:			Union[str, Any]						=						list(Path(model.output_dir    ).glob('''**/*.bin'''    )    )
					A_					:			Union[str, Any]						=						list(Path(model.output_dir    ).glob('''best_tfmr/*'''    )    )
					assert len(_lowerCamelCase    ) > 2
					self.assertEqual(len(_lowerCamelCase    ) ,       2    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			str						=						{
					    '''student_encoder_layers''': 1,
					    '''student_decoder_layers''': 1,
					    '''alpha_hid''': 2.0,
					    '''teacher''': T5_TINY,
					    '''model_name_or_path''': T5_TINY,
					    '''tokenizer_name''': T5_TINY,
					}
					self._test_distiller_cli(_lowerCamelCase    )







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Dict						=						{
					    '''teacher''': T5_TINY,
					    '''student''': T5_TINIER,
					    '''model_name_or_path''': T5_TINIER,
					    '''tokenizer_name''': T5_TINIER,
					}
					self._test_distiller_cli(_lowerCamelCase    )







				def    a_					(							self				:   str ,       _lowerCamelCase				:   List[str] ,       _lowerCamelCase				:   Optional[Any]=True    ):



					"""simple docstring"""

					A_					:			Any						=						{
					    '''label_smoothing''': 0.0,
					    '''early_stopping_patience''': -1,
					    '''train_batch_size''': 1,
					    '''eval_batch_size''': 2,
					    '''max_epochs''': 2,
					    '''alpha_mlm''': 0.2,
					    '''alpha_ce''': 0.8,
					    '''do_predict''': True,
					    '''model_name_or_path''': '''sshleifer/tinier_bart''',
					    '''teacher''': CHEAP_ARGS['''model_name_or_path'''],
					    '''val_check_interval''': 0.5,
					}
					default_updates.update(_lowerCamelCase    )
					A_					:			dict						=						CHEAP_ARGS.copy()
					A_					:			int						=						make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()    )
					A_					:			Dict						=						self.get_auto_remove_tmp_dir()

					args_d.update(data_dir=_lowerCamelCase ,       output_dir=_lowerCamelCase ,       **_lowerCamelCase    )
					A_					:			List[Any]						=						distill_main(argparse.Namespace(**_lowerCamelCase    )    )
					if not check_contents:
						return model
					A_					:			Dict						=						os.listdir(_lowerCamelCase    )
					A_					:			Optional[Any]						=						{os.path.basename(_lowerCamelCase    ) for p in contents}
					A_					:			Union[str, Any]						=						[p for p in contents if p.endswith('''ckpt'''    )]
					assert len(_lowerCamelCase    ) > 0

					self.assertIn('''test_generations.txt''' ,       _lowerCamelCase    )
					self.assertIn('''test_results.txt''' ,       _lowerCamelCase    )

					A_					:			Optional[Any]						=						load_json(model.metrics_save_path    )
					A_					:			Optional[int]						=						metrics['''val'''][-1]
					self.assertGreaterEqual(last_step_stats['''val_avg_gen_time'''] ,       0.01    )
					self.assertGreaterEqual(1.0 ,       last_step_stats['''val_avg_gen_time''']    )
					self.assertIsInstance(last_step_stats[F"""val_avg_{model.val_metric}"""] ,       _lowerCamelCase    )
					A_					:			Dict						=						int(args_d['''max_epochs'''] * (1 / args_d['''val_check_interval''']) + 1    )
					self.assertEqual(len(metrics['''val''']    ) ,       _lowerCamelCase    )
					self.assertEqual(len(metrics['''test''']    ) ,       1    )
					return model

class     lowercase (      __UpperCAmelCase):







				@parameterized.expand(
				    [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY] ,       )
				def    a_					(							self				:   Dict ,       _lowerCamelCase				:   Any    ):



					"""simple docstring"""

					A_					:			dict						=						CHEAP_ARGS.copy()
					A_					:			str						=						'''translation''' if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else '''summarization'''
					A_					:			Any						=						0.1 if task == '''translation''' else 0

					A_					:			List[str]						=						make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()    )
					A_					:			Optional[Any]						=						self.get_auto_remove_tmp_dir()
					args_d.update(
					    data_dir=_lowerCamelCase ,       model_name_or_path=_lowerCamelCase ,       tokenizer_name=_lowerCamelCase ,       train_batch_size=2 ,       eval_batch_size=2 ,       output_dir=_lowerCamelCase ,       do_predict=_lowerCamelCase ,       task=_lowerCamelCase ,       src_lang='''en_XX''' ,       tgt_lang='''ro_RO''' ,       freeze_encoder=_lowerCamelCase ,       freeze_embeds=_lowerCamelCase ,       )
					assert "n_train" in args_d
					A_					:			List[Any]						=						argparse.Namespace(**_lowerCamelCase    )
					A_					:			List[str]						=						main(_lowerCamelCase    )

					A_					:			Any						=						module.model.get_input_embeddings()
					assert not input_embeds.weight.requires_grad
					if model == T5_TINY:
						A_					:			Optional[int]						=						module.model.lm_head
						assert not lm_head.weight.requires_grad
						assert (lm_head.weight == input_embeds.weight).all().item()
					elif model == FSMT_TINY:
						A_					:			str						=						module.model.model
						A_					:			Any						=						fsmt.decoder.embed_positions
						assert not embed_pos.weight.requires_grad
						assert not fsmt.decoder.embed_tokens.weight.requires_grad
						# check that embeds are not the same
						assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
					else:
						A_					:			int						=						module.model.model
						A_					:			str						=						bart.decoder.embed_positions
						assert not embed_pos.weight.requires_grad
						assert not bart.shared.weight.requires_grad
						# check that embeds are the same
						assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
						assert bart.decoder.embed_tokens == bart.shared

					A_					:			Optional[int]						=						load_json(module.output_dir / '''text_batch.json'''    )
					assert isinstance(_lowerCamelCase ,       _lowerCamelCase    )
					assert len(_lowerCamelCase    ) >= 4







				def    a_					(							self				:   List[str]    ):



					"""simple docstring"""

					A_					:			dict						=						CHEAP_ARGS.copy()

					A_					:			List[Any]						=						'''summarization'''
					A_					:			Union[str, Any]						=						make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()    )

					args_d.update(
					    data_dir=_lowerCamelCase ,       tokenizer_name=_lowerCamelCase ,       train_batch_size=2 ,       eval_batch_size=2 ,       do_predict=_lowerCamelCase ,       task=_lowerCamelCase ,       src_lang='''en_XX''' ,       tgt_lang='''ro_RO''' ,       freeze_encoder=_lowerCamelCase ,       freeze_embeds=_lowerCamelCase ,       )

					# test models whose config includes the extra_model_args
					A_					:			str						=						BART_TINY
					A_					:			List[Any]						=						self.get_auto_remove_tmp_dir()
					A_					:			Union[str, Any]						=						args_d.copy()
					args_da.update(
					    model_name_or_path=_lowerCamelCase ,       output_dir=_lowerCamelCase ,       )
					A_					:			Dict						=						('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
					for p in extra_model_params:
						A_					:			int						=						0.5
					A_					:			Any						=						argparse.Namespace(**_lowerCamelCase    )
					A_					:			int						=						main(_lowerCamelCase    )
					for p in extra_model_params:
						assert getattr(model.config ,       _lowerCamelCase    ) == 0.5, F"""failed to override the model config for param {p}"""

					# test models whose config doesn't include the extra_model_args
					A_					:			Any						=						T5_TINY
					A_					:			Any						=						self.get_auto_remove_tmp_dir()
					A_					:			Optional[Any]						=						args_d.copy()
					args_da.update(
					    model_name_or_path=_lowerCamelCase ,       output_dir=_lowerCamelCase ,       )
					A_					:			List[str]						=						'''encoder_layerdrop'''
					A_					:			Any						=						0.5
					A_					:			str						=						argparse.Namespace(**_lowerCamelCase    )
					with pytest.raises(_lowerCamelCase    ) as excinfo:
						A_					:			Optional[int]						=						main(_lowerCamelCase    )
					assert str(excinfo.value    ) == F"""model config doesn't have a `{unsupported_param}` attribute"""







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			dict						=						CHEAP_ARGS.copy()

					A_					:			Union[str, Any]						=						'''summarization'''
					A_					:			Union[str, Any]						=						make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()    )

					A_					:			List[str]						=						BART_TINY
					A_					:			int						=						self.get_auto_remove_tmp_dir()

					args_d.update(
					    data_dir=_lowerCamelCase ,       model_name_or_path=_lowerCamelCase ,       output_dir=_lowerCamelCase ,       tokenizer_name=_lowerCamelCase ,       train_batch_size=2 ,       eval_batch_size=2 ,       do_predict=_lowerCamelCase ,       task=_lowerCamelCase ,       src_lang='''en_XX''' ,       tgt_lang='''ro_RO''' ,       freeze_encoder=_lowerCamelCase ,       freeze_embeds=_lowerCamelCase ,       )

					# emulate finetune.py
					A_					:			Dict						=						argparse.ArgumentParser()
					A_					:			Optional[Any]						=						pl.Trainer.add_argparse_args(_lowerCamelCase    )
					A_					:			Tuple						=						SummarizationModule.add_model_specific_args(_lowerCamelCase ,       os.getcwd()    )
					A_					:			int						=						{'''--help''': True}

					# --help test
					with pytest.raises(_lowerCamelCase    ) as excinfo:
						with CaptureStdout() as cs:
							A_					:			Tuple						=						parser.parse_args(_lowerCamelCase    )
						assert False, "--help is expected to sys.exit"
					assert excinfo.type == SystemExit
					A_					:			Optional[Any]						=						lightning_base.arg_to_scheduler_metavar
					assert expected in cs.out, "--help is expected to list the supported schedulers"

					# --lr_scheduler=non_existing_scheduler test
					A_					:			List[str]						=						'''non_existing_scheduler'''
					A_					:			List[Any]						=						{F"""--lr_scheduler={unsupported_param}"""}
					with pytest.raises(_lowerCamelCase    ) as excinfo:
						with CaptureStderr() as cs:
							A_					:			str						=						parser.parse_args(_lowerCamelCase    )
						assert False, "invalid argument is expected to sys.exit"
					assert excinfo.type == SystemExit
					A_					:			List[Any]						=						F"""invalid choice: '{unsupported_param}'"""
					assert expected in cs.err, F"""should have bailed on invalid choice of scheduler {unsupported_param}"""

					# --lr_scheduler=existing_scheduler test
					A_					:			Union[str, Any]						=						'''cosine'''
					A_					:			Optional[int]						=						args_d.copy()
					A_					:			Optional[int]						=						supported_param
					A_					:			List[Any]						=						argparse.Namespace(**_lowerCamelCase    )
					A_					:			int						=						main(_lowerCamelCase    )
					assert (
					    getattr(model.hparams ,       '''lr_scheduler'''    ) == supported_param
					), F"""lr_scheduler={supported_param} shouldn't fail"""











"""simple docstring"""

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     lowercase (      __UpperCAmelCase       ,		unittest.TestCase):
				__lowerCAmelCase      :      Any					    =				ElectraTokenizer
				__lowerCAmelCase      :      Optional[int]					    =				ElectraTokenizerFast
				__lowerCAmelCase      :      Any					    =				True
				__lowerCAmelCase      :      Tuple					    =				True
				__lowerCAmelCase      :      Optional[Any]					    =				filter_non_english







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					super().setUp()

					A_					:			int						=						[
					    '''[UNK]''',
					    '''[CLS]''',
					    '''[SEP]''',
					    '''[PAD]''',
					    '''[MASK]''',
					    '''want''',
					    '''##want''',
					    '''##ed''',
					    '''wa''',
					    '''un''',
					    '''runn''',
					    '''##ing''',
					    ''',''',
					    '''low''',
					    '''lowest''',
					]
					A_					:			str						=						os.path.join(self.tmpdirname ,       VOCAB_FILES_NAMES['''vocab_file''']    )
					with open(self.vocab_file ,       '''w''' ,       encoding='''utf-8'''    ) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						'''UNwant\u00E9d,running'''
					A_					:			Optional[int]						=						'''unwanted, running'''
					return input_text, output_text







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Tuple						=						self.tokenizer_class(self.vocab_file    )

					A_					:			Dict						=						tokenizer.tokenize('''UNwant\u00E9d,running'''    )
					self.assertListEqual(_lowerCamelCase ,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase    ) ,       [9, 6, 7, 12, 10, 11]    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					if not self.test_rust_tokenizer:
						return

					A_					:			Any						=						self.get_tokenizer()
					A_					:			Optional[Any]						=						self.get_rust_tokenizer()

					A_					:			Union[str, Any]						=						'''UNwant\u00E9d,running'''

					A_					:			Optional[int]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			int						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Any						=						self.get_rust_tokenizer()
					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			List[Any]						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					# With lower casing
					A_					:			List[str]						=						self.get_tokenizer(do_lower_case=_lowerCamelCase    )
					A_					:			Any						=						self.get_rust_tokenizer(do_lower_case=_lowerCamelCase    )

					A_					:			List[Any]						=						'''UNwant\u00E9d,running'''

					A_					:			List[str]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			List[str]						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[Any]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			List[Any]						=						self.get_rust_tokenizer()
					A_					:			List[Any]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			Dict						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Union[str, Any]						=						BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    ) ,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )







				def    a_					(							self				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''h\u00E9llo''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Tuple						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			int						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       never_split=['''[UNK]''']    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

					A_					:			Any						=						{}
					for i, token in enumerate(_lowerCamelCase    ):
						A_					:			int						=						i
					A_					:			str						=						WordpieceTokenizer(vocab=_lowerCamelCase ,       unk_token='''[UNK]'''    )

					self.assertListEqual(tokenizer.tokenize(''''''    ) ,       []    )

					self.assertListEqual(tokenizer.tokenize('''unwanted running'''    ) ,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

					self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    ) ,       ['''[UNK]''', '''runn''', '''##ing''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_whitespace(''' '''    )    )
					self.assertTrue(_is_whitespace('''\t'''    )    )
					self.assertTrue(_is_whitespace('''\r'''    )    )
					self.assertTrue(_is_whitespace('''\n'''    )    )
					self.assertTrue(_is_whitespace('''\u00A0'''    )    )

					self.assertFalse(_is_whitespace('''A'''    )    )
					self.assertFalse(_is_whitespace('''-'''    )    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					self.assertTrue(_is_control('''\u0005'''    )    )

					self.assertFalse(_is_control('''A'''    )    )
					self.assertFalse(_is_control(''' '''    )    )
					self.assertFalse(_is_control('''\t'''    )    )
					self.assertFalse(_is_control('''\r'''    )    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_punctuation('''-'''    )    )
					self.assertTrue(_is_punctuation('''$'''    )    )
					self.assertTrue(_is_punctuation('''`'''    )    )
					self.assertTrue(_is_punctuation('''.'''    )    )

					self.assertFalse(_is_punctuation('''A'''    )    )
					self.assertFalse(_is_punctuation(''' '''    )    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						self.get_tokenizer()
					A_					:			int						=						self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )







				@slow
				def    a_					(							self				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''    )

					A_					:			Tuple						=						tokenizer.encode('''sequence builders''' ,       add_special_tokens=_lowerCamelCase    )
					A_					:			List[Any]						=						tokenizer.encode('''multi-sequence build''' ,       add_special_tokens=_lowerCamelCase    )

					A_					:			List[Any]						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase    )
					A_					:			Any						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase ,       _lowerCamelCase    )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			Optional[Any]						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[int]						=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							A_					:			Optional[int]						=						tokenizer_r.encode_plus(
							    _lowerCamelCase ,       return_attention_mask=_lowerCamelCase ,       return_token_type_ids=_lowerCamelCase ,       return_offsets_mapping=_lowerCamelCase ,       add_special_tokens=_lowerCamelCase ,       )

							A_					:			Union[str, Any]						=						tokenizer_r.do_lower_case if hasattr(_lowerCamelCase ,       '''do_lower_case'''    ) else False
							A_					:			List[Any]						=						(
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''A'''),
							        ((1, 2), ''','''),
							        ((3, 5), '''na'''),
							        ((5, 6), '''##ï'''),
							        ((6, 8), '''##ve'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''Allen'''),
							        ((21, 23), '''##NL'''),
							        ((23, 24), '''##P'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''a'''),
							        ((1, 2), ''','''),
							        ((3, 8), '''naive'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''allen'''),
							        ((21, 23), '''##nl'''),
							        ((23, 24), '''##p'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results] ,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
							self.assertEqual([e[0] for e in expected_results] ,       tokens['''offset_mapping''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''的''', '''人''', '''有''']
					A_					:			Optional[int]						=						''''''.join(_lowerCamelCase    )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			List[Any]						=						True
							A_					:			Any						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			str						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			List[Any]						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			Any						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			Union[str, Any]						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

							A_					:			List[Any]						=						False
							A_					:			Any						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			Union[str, Any]						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[Any]						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			int						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			str						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that only the first Chinese character is not preceded by "##".
							A_					:			Dict						=						[
							    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase    )
							]
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )





