





import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		)
class 		UpperCAmelCase       :







						'''simple docstring'''





						def __init__(						self							:					Optional[Any]	,     __lowercase							:					Optional[int]	,     __lowercase							:					Tuple=13	,     __lowercase							:					Any=7	,     __lowercase							:					List[Any]=True	,     __lowercase							:					Optional[int]=True	,     __lowercase							:					List[str]=True	,     __lowercase							:					Dict=True	,     __lowercase							:					Dict=99	,     __lowercase							:					int=[1, 1, 2]	,     __lowercase							:					Optional[int]=1	,     __lowercase							:					Optional[int]=32	,     __lowercase							:					List[str]=4	,     __lowercase							:					List[Any]=8	,     __lowercase							:					List[Any]=37	,     __lowercase							:					Any="gelu_new"	,     __lowercase							:					Optional[Any]=0.1	,     __lowercase							:					Tuple=0.1	,     __lowercase							:					Optional[int]=0.0	,     __lowercase							:					Any=5_12	,     __lowercase							:					Union[str, Any]=3	,     __lowercase							:					Tuple=0.02	,     __lowercase							:					Union[str, Any]=3	,     __lowercase							:					Optional[int]=4	,     __lowercase							:					Union[str, Any]=None	,     __lowercase							:					List[str]=False	,     ):






							"""simple docstring"""






							snake_case_		   =					parent
							snake_case_		   =					batch_size
							snake_case_		   =					seq_length
							snake_case_		   =					is_training
							snake_case_		   =					use_input_mask
							snake_case_		   =					use_token_type_ids
							snake_case_		   =					use_labels
							snake_case_		   =					vocab_size
							snake_case_		   =					block_sizes
							snake_case_		   =					num_decoder_layers
							snake_case_		   =					d_model
							snake_case_		   =					n_head
							snake_case_		   =					d_head
							snake_case_		   =					d_inner
							snake_case_		   =					hidden_act
							snake_case_		   =					hidden_dropout
							snake_case_		   =					attention_dropout
							snake_case_		   =					activation_dropout
							snake_case_		   =					max_position_embeddings
							snake_case_		   =					type_vocab_size
							snake_case_		   =					2
							snake_case_		   =					num_labels
							snake_case_		   =					num_choices
							snake_case_		   =					scope
							snake_case_		   =					initializer_std

							# Used in the tests to check the size of the first attention layer
							snake_case_		   =					n_head
							# Used in the tests to check the size of the first hidden state
							snake_case_		   =					self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							snake_case_		   =					sum(self.block_sizes						) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
								snake_case_		   =					self.num_hidden_layers + 2






						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.vocab_size						)

							snake_case_		   =					None
							if self.use_input_mask:
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     vocab_size=2						)

							snake_case_		   =					None
							if self.use_token_type_ids:
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.type_vocab_size						)

							snake_case_		   =					None
							snake_case_		   =					None
							snake_case_		   =					None
							if self.use_labels:
								snake_case_		   =					ids_tensor([self.batch_size]	,     self.type_sequence_label_size						)
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.num_labels						)
								snake_case_		   =					ids_tensor([self.batch_size]	,     self.num_choices						)
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     1						)

							snake_case_		   =					self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    fake_token_labels,
							)






						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							return FunnelConfig(
							    vocab_size=self.vocab_size	,     block_sizes=self.block_sizes	,     num_decoder_layers=self.num_decoder_layers	,     d_model=self.d_model	,     n_head=self.n_head	,     d_head=self.d_head	,     d_inner=self.d_inner	,     hidden_act=self.hidden_act	,     hidden_dropout=self.hidden_dropout	,     attention_dropout=self.attention_dropout	,     activation_dropout=self.activation_dropout	,     max_position_embeddings=self.max_position_embeddings	,     type_vocab_size=self.type_vocab_size	,     initializer_std=self.initializer_std	,     )






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					List[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					List[str]	,     __lowercase							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					str	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Any	,     __lowercase							:					int	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					str	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelBaseModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 2, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 3, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 2, self.d_model)						)






						def 	snake_case__						(						self							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForPreTraining(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length)						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					str	,     __lowercase							:					List[str]	,     __lowercase							:					Any	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					int	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelForMaskedLM(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length, self.vocab_size)						)






						def 	snake_case__						(						self							:					Any	,     __lowercase							:					Dict	,     __lowercase							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForSequenceClassification(__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.num_labels)						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					str	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     __lowercase							:					List[str]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_choices
							snake_case_		   =					FunnelForMultipleChoice(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					input_ids.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					token_type_ids.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					input_mask.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					model(
							    __lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase	,     )
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.num_choices)						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[int]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					int	,     __lowercase							:					Dict	,     __lowercase							:					List[str]	,     __lowercase							:					Tuple	,     __lowercase							:					List[str]	,     __lowercase							:					List[Any]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForTokenClassification(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length, self.num_labels)						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					Any	,     __lowercase							:					List[Any]	,     __lowercase							:					List[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[int]	,     __lowercase							:					Optional[int]	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelForQuestionAnswering(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(
							    __lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     start_positions=__lowercase	,     end_positions=__lowercase	,     )
							self.parent.assertEqual(result.start_logits.shape	,     (self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape	,     (self.batch_size, self.seq_length)						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					self.prepare_config_and_inputs()
							(
							    (
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,				(
							    snake_case_				
							)   ,								
							)		   =					config_and_inputs
							snake_case_		   =					{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class 		UpperCAmelCase       (    UpperCAmelCase__	,       UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCAmelCase_							=       (
						    {
						        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
						        '''fill-mask''': FunnelForMaskedLM,
						        '''question-answering''': FunnelForQuestionAnswering,
						        '''text-classification''': FunnelForSequenceClassification,
						        '''token-classification''': FunnelForTokenClassification,
						        '''zero-shot''': FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)






						def 	snake_case__						(						self							:					Dict	,     __lowercase							:					Any	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Optional[int]=False						):






							"""simple docstring"""






							snake_case_		   =					super()._prepare_for_class(__lowercase	,     __lowercase	,     return_labels=__lowercase						)

							if return_labels:
								if model_class in get_values(__lowercase						):
									snake_case_		   =					torch.zeros(
									    (self.model_tester.batch_size, self.model_tester.seq_length)	,     dtype=torch.long	,     device=__lowercase						)
							return inputs_dict






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					FunnelModelTester(self						)
							snake_case_		   =					ConfigTester(self	,     config_class=__lowercase						)






						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*__lowercase						)






						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__lowercase						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowercase						)






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowercase						)






						def 	snake_case__						(						self							:					List[Any]	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							if hasattr(__lowercase	,     "weight"						) and module.weight is not None:
								module.weight.data.fill_(3						)
							if hasattr(__lowercase	,     "bias"						) and module.bias is not None:
								module.bias.data.fill_(3						)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__lowercase	,     __lowercase						) and getattr(__lowercase	,     __lowercase						) is not None:
									snake_case_		   =					getattr(__lowercase	,     __lowercase						)
									weight.data.fill_(3						)







@require_torch
class 		UpperCAmelCase       (    UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					FunnelModelTester(self	,     base=__lowercase						)
							snake_case_		   =					ConfigTester(self	,     config_class=__lowercase						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def 	snake_case__						(						self							:					Optional[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*__lowercase						)






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__lowercase						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__lowercase						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_   ,				snake_case_		   =					self.model_tester.prepare_config_and_inputs_for_common()
							snake_case_		   =					True

							for model_class in self.all_model_classes:
								if model_class.__name__ == "FunnelBaseModel":
									continue
								snake_case_		   =					model_class(__lowercase						)
								model.to(__lowercase						)
								model.train()
								snake_case_		   =					self._prepare_for_class(__lowercase	,     __lowercase	,     return_labels=__lowercase						)
								snake_case_		   =					model(**__lowercase						).loss
								loss.backward()






						def 	snake_case__						(						self							:					str	,     __lowercase							:					Optional[Any]						):






							"""simple docstring"""






							if hasattr(__lowercase	,     "weight"						) and module.weight is not None:
								module.weight.data.fill_(3						)
							if hasattr(__lowercase	,     "bias"						) and module.bias is not None:
								module.bias.data.fill_(3						)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__lowercase	,     __lowercase						) and getattr(__lowercase	,     __lowercase						) is not None:
									snake_case_		   =					getattr(__lowercase	,     __lowercase						)
									weight.data.fill_(3						)







@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase       (    unittest.TestCase      ):







						'''simple docstring'''





						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					13
							snake_case_		   =					7
							snake_case_		   =					torch.arange(0	,     batch_size * sequence_length						).long().reshape(__lowercase	,     __lowercase						)
							snake_case_		   =					[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
							snake_case_		   =					torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]						)

							snake_case_		   =					FunnelModel.from_pretrained("sgugger/funnel-random-tiny"						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)[0].abs()

							snake_case_		   =					torch.tensor(2344.8352						)
							snake_case_		   =					torch.tensor(0.8052						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)

							snake_case_		   =					torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]						)
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)[0].abs()

							snake_case_		   =					torch.tensor(2343.8425						)
							snake_case_		   =					torch.tensor(0.8049						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)






						@slow
						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					FunnelTokenizer.from_pretrained("huggingface/funnel-small"						)
							snake_case_		   =					FunnelModel.from_pretrained("huggingface/funnel-small"						)
							snake_case_		   =					tokenizer("Hello! I am the Funnel Transformer model."	,     return_tensors="pt"						)
							snake_case_		   =					model(**__lowercase						)[0]

							snake_case_		   =					torch.tensor(235.7246						)
							snake_case_		   =					torch.tensor(0.0256						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase__    :					List[str]					=						logging.get_logger(__name__)

lowercase__    :					Tuple					=						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowercase__    :					Any					=						{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

lowercase__    :					Tuple					=						{
    "openai-gpt": 512,
}


def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					text.strip()
	if not text:
		return []
	snake_case_		   =					text.split()
	return tokens
class 		UpperCAmelCase       (    UpperCAmelCase__      ):







						'''simple docstring'''





						def __init__(						self							:					Dict	,     __lowercase							:					str=True	,     __lowercase							:					Union[str, Any]=None	,     __lowercase							:					str=True	,     __lowercase							:					Dict=None	,     __lowercase							:					str=True	,     ):






							"""simple docstring"""






							if never_split is None:
								snake_case_		   =					[]
							snake_case_		   =					do_lower_case
							snake_case_		   =					set(__lowercase						)
							snake_case_		   =					tokenize_chinese_chars
							snake_case_		   =					strip_accents
							snake_case_		   =					do_split_on_punc






						def 	snake_case__						(						self							:					Any	,     __lowercase							:					Any	,     __lowercase							:					Optional[int]=None						):






							"""simple docstring"""






							snake_case_		   =					self.never_split.union(set(__lowercase						)						) if never_split else self.never_split
							snake_case_		   =					self._clean_text(__lowercase						)

							# This was added on November 1st, 2018 for the multilingual and Chinese
							# models. This is also applied to the English models now, but it doesn't
							# matter since the English models were not trained on any Chinese data
							# and generally don't have any Chinese data in them (there are Chinese
							# characters in the vocabulary because Wikipedia does have some Chinese
							# words in the English Wikipedia.).
							if self.tokenize_chinese_chars:
								snake_case_		   =					self._tokenize_chinese_chars(__lowercase						)
							# prevents treating the same character with different unicode codepoints as different characters
							snake_case_		   =					unicodedata.normalize("NFC"	,     __lowercase						)
							snake_case_		   =					whitespace_tokenize(__lowercase						)
							snake_case_		   =					[]
							for token in orig_tokens:
								if token not in never_split:
									if self.do_lower_case:
										snake_case_		   =					token.lower()
										if self.strip_accents is not False:
											snake_case_		   =					self._run_strip_accents(__lowercase						)
									elif self.strip_accents:
										snake_case_		   =					self._run_strip_accents(__lowercase						)
								split_tokens.extend(self._run_split_on_punc(__lowercase	,     __lowercase						)						)

							snake_case_		   =					whitespace_tokenize(" ".join(__lowercase						)						)
							return output_tokens






						def 	snake_case__						(						self							:					str	,     __lowercase							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					unicodedata.normalize("NFD"	,     __lowercase						)
							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					unicodedata.category(__lowercase						)
								if cat == "Mn":
									continue
								output.append(__lowercase						)
							return "".join(__lowercase						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[int]=None						):






							"""simple docstring"""






							if not self.do_split_on_punc or (never_split is not None and text in never_split):
								return [text]
							snake_case_		   =					list(__lowercase						)
							snake_case_		   =					0
							snake_case_		   =					True
							snake_case_		   =					[]
							while i < len(__lowercase						):
								snake_case_		   =					chars[i]
								if _is_punctuation(__lowercase						):
									output.append([char]						)
									snake_case_		   =					True
								else:
									if start_new_word:
										output.append([]						)
									snake_case_		   =					False
									output[-1].append(__lowercase						)
								i += 1

							return ["".join(__lowercase						) for x in output]






						def 	snake_case__						(						self							:					Optional[int]	,     __lowercase							:					str						):






							"""simple docstring"""






							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					ord(__lowercase						)
								if self._is_chinese_char(__lowercase						):
									output.append(" "						)
									output.append(__lowercase						)
									output.append(" "						)
								else:
									output.append(__lowercase						)
							return "".join(__lowercase						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					str						):






							"""simple docstring"""






							if (
							    (cp >= 0X4_e00 and cp <= 0X9_fff)
							    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
							    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
							    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
							    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
							    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
							    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
							    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
							):  #
								return True

							return False






						def 	snake_case__						(						self							:					int	,     __lowercase							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					ord(__lowercase						)
								if cp == 0 or cp == 0Xf_ffd or _is_control(__lowercase						):
									continue
								if _is_whitespace(__lowercase						):
									output.append(" "						)
								else:
									output.append(__lowercase						)
							return "".join(__lowercase						)







def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					set()
	snake_case_		   =					word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)							)
		snake_case_		   =					char
	return pairs


def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					text.replace("—"					,     "-"							)
	snake_case_		   =					text.replace("–"					,     "-"							)
	snake_case_		   =					text.replace("―"					,     "-"							)
	snake_case_		   =					text.replace("…"					,     "..."							)
	snake_case_		   =					text.replace("´"					,     "'"							)
	snake_case_		   =					re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"					,     R" \1 "					,     _A							)
	snake_case_		   =					re.sub(R"\s*\n\s*"					,     " \n "					,     _A							)
	snake_case_		   =					re.sub(R"[^\S\n]+"					,     " "					,     _A							)
	return text.strip()






class 		UpperCAmelCase       (    UpperCAmelCase__      ):







						'''simple docstring'''





						lowerCAmelCase_							=       VOCAB_FILES_NAMES
						lowerCAmelCase_							=       PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase_							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase_							=       ['''input_ids''', '''attention_mask''']






						def __init__(						self							:					Dict	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Tuple	,     __lowercase							:					Any="<unk>"	,     **__lowercase							:					List[Any]						):






							"""simple docstring"""






							super().__init__(unk_token=__lowercase	,     **__lowercase						)

							try:
								import ftfy
								from spacy.lang.en import English

								snake_case_		   =					English()
								snake_case_		   =					_nlp.tokenizer
								snake_case_		   =					ftfy.fix_text
							except ImportError:
								logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
								snake_case_		   =					BasicTokenizer(do_lower_case=__lowercase						)
								snake_case_		   =					None

							with open(__lowercase	,     encoding="utf-8"						) as vocab_handle:
								snake_case_		   =					json.load(__lowercase						)
							snake_case_		   =					{v: k for k, v in self.encoder.items()}
							with open(__lowercase	,     encoding="utf-8"						) as merges_handle:
								snake_case_		   =					merges_handle.read().split("\n"						)[1:-1]
							snake_case_		   =					[tuple(merge.split()						) for merge in merges]
							snake_case_		   =					dict(zip(__lowercase	,     range(len(__lowercase						)						)						)						)
							snake_case_		   =					{}






						@property
						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							return True






						@property
						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							return len(self.encoder						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							return dict(self.encoder	,     **self.added_tokens_encoder						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					int						):






							"""simple docstring"""






							snake_case_		   =					tuple(token[:-1]						) + (token[-1] + "</w>",)
							if token in self.cache:
								return self.cache[token]
							snake_case_		   =					get_pairs(__lowercase						)

							if not pairs:
								return token + "</w>"

							while True:
								snake_case_		   =					min(__lowercase	,     key=lambda __lowercase						: self.bpe_ranks.get(__lowercase	,     float("inf"						)						)						)
								if bigram not in self.bpe_ranks:
									break
								snake_case_   ,				snake_case_		   =					bigram
								snake_case_		   =					[]
								snake_case_		   =					0
								while i < len(__lowercase						):
									try:
										snake_case_		   =					word.index(__lowercase	,     __lowercase						)
									except ValueError:
										new_word.extend(word[i:]						)
										break
									else:
										new_word.extend(word[i:j]						)
										snake_case_		   =					j

									if word[i] == first and i < len(__lowercase						) - 1 and word[i + 1] == second:
										new_word.append(first + second						)
										i += 2
									else:
										new_word.append(word[i]						)
										i += 1
								snake_case_		   =					tuple(__lowercase						)
								snake_case_		   =					new_word
								if len(__lowercase						) == 1:
									break
								else:
									snake_case_		   =					get_pairs(__lowercase						)
							snake_case_		   =					" ".join(__lowercase						)
							if word == "\n  </w>":
								snake_case_		   =					"\n</w>"
							snake_case_		   =					word
							return word






						def 	snake_case__						(						self							:					List[str]	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							snake_case_		   =					[]
							if self.fix_text is None:
								# Using BERT's BasicTokenizer
								snake_case_		   =					self.nlp.tokenize(__lowercase						)
								for token in text:
									split_tokens.extend(list(self.bpe(__lowercase						).split(" "						)						)						)
							else:
								# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
								snake_case_		   =					self.nlp(text_standardize(self.fix_text(__lowercase						)						)						)
								for token in text:
									split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
							return split_tokens






						def 	snake_case__						(						self							:					int	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							return self.encoder.get(__lowercase	,     self.encoder.get(self.unk_token						)						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Any						):






							"""simple docstring"""






							return self.decoder.get(__lowercase	,     self.unk_token						)






						def 	snake_case__						(						self							:					str	,     __lowercase							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					"".join(__lowercase						).replace("</w>"	,     " "						).strip()
							return out_string






						def 	snake_case__						(						self							:					int	,     __lowercase							:					str	,     __lowercase							:					Optional[str] = None						):






							"""simple docstring"""






							if not os.path.isdir(__lowercase						):
								logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
								return
							snake_case_		   =					os.path.join(
							    __lowercase	,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
							snake_case_		   =					os.path.join(
							    __lowercase	,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

							with open(__lowercase	,     "w"	,     encoding="utf-8"						) as f:
								f.write(json.dumps(self.encoder	,     indent=2	,     sort_keys=__lowercase	,     ensure_ascii=__lowercase						) + "\n"						)

							snake_case_		   =					0
							with open(__lowercase	,     "w"	,     encoding="utf-8"						) as writer:
								writer.write("#version: 0.2\n"						)
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,     key=lambda __lowercase						: kv[1]						):
									if index != token_index:
										logger.warning(
										    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
										    " Please check that the tokenizer is not corrupted!"						)
										snake_case_		   =					token_index
									writer.write(" ".join(__lowercase						) + "\n"						)
									index += 1

							return vocab_file, merge_file
