





'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__UpperCAmelCase         =   logging.get_logger(__name__)

__UpperCAmelCase         =   {"""vocab_file""": """vocab.txt"""}

__UpperCAmelCase         =   {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

__UpperCAmelCase         =   {
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

__UpperCAmelCase         =   {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}




class       UpperCamelCase__			(			lowercase_	):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__						=					VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE__						=					SplinterTokenizer


					def __init__(		self							:	int     ,		lowerCamelCase_							:	str=None     ,		lowerCamelCase_							:	Tuple=None     ,		lowerCamelCase_							:	Tuple=True     ,		lowerCamelCase_							:	str="[UNK]"     ,		lowerCamelCase_							:	int="[SEP]"     ,		lowerCamelCase_							:	List[str]="[PAD]"     ,		lowerCamelCase_							:	List[Any]="[CLS]"     ,		lowerCamelCase_							:	Optional[Any]="[MASK]"     ,		lowerCamelCase_							:	List[str]="[QUESTION]"     ,		lowerCamelCase_							:	str=True     ,		lowerCamelCase_							:	Tuple=None     ,		**lowerCamelCase_							:	Tuple     ,		):

							'''simple docstring'''



							super().__init__(
							    lowerCamelCase_     ,		tokenizer_file=lowerCamelCase_     ,		do_lower_case=lowerCamelCase_     ,		unk_token=lowerCamelCase_     ,		sep_token=lowerCamelCase_     ,		pad_token=lowerCamelCase_     ,		cls_token=lowerCamelCase_     ,		mask_token=lowerCamelCase_     ,		tokenize_chinese_chars=lowerCamelCase_     ,		strip_accents=lowerCamelCase_     ,		additional_special_tokens=(question_token,)     ,		**lowerCamelCase_     ,		)

							SCREAMING_SNAKE_CASE				:							str						=	json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
							if (
							    pre_tok_state.get("""lowercase"""     ,		lowerCamelCase_		) != do_lower_case
							    or pre_tok_state.get("""strip_accents"""     ,		lowerCamelCase_		) != strip_accents
							):
									SCREAMING_SNAKE_CASE				:							Optional[int]						=	getattr(lowerCamelCase_     ,		pre_tok_state.pop("""type"""		)		)
									SCREAMING_SNAKE_CASE				:							int						=	do_lower_case
									SCREAMING_SNAKE_CASE				:							int						=	strip_accents
									SCREAMING_SNAKE_CASE				:							Dict						=	pre_tok_class(**lowerCamelCase_		)

							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	do_lower_case


					@property
					def lowerCamelCase_	(		self							:	Any		):

							'''simple docstring'''



							return self.convert_tokens_to_ids(self.question_token		)


					def lowerCamelCase_	(		self							:	List[Any]     ,		lowerCamelCase_							:	List[int]     ,		lowerCamelCase_							:	Optional[List[int]] = None		):

							'''simple docstring'''



							if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							SCREAMING_SNAKE_CASE				:							Optional[int]						=	[self.cls_token_id]
							SCREAMING_SNAKE_CASE				:							List[str]						=	[self.sep_token_id]
							SCREAMING_SNAKE_CASE				:							int						=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""		)]
							if self.padding_side == "right":
									# Input is question-then-context
									return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
							else:
									# Input is context-then-question
									return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


					def lowerCamelCase_	(		self							:	int     ,		lowerCamelCase_							:	List[int]     ,		lowerCamelCase_							:	Optional[List[int]] = None		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	[self.sep_token_id]
							SCREAMING_SNAKE_CASE				:							List[str]						=	[self.cls_token_id]
							SCREAMING_SNAKE_CASE				:							Tuple						=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""		)]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep		) * [0]

							if self.padding_side == "right":
									# Input is question-then-context
									return len(cls + token_ids_a + question_suffix + sep		) * [0] + len(token_ids_a + sep		) * [1]
							else:
									# Input is context-then-question
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + question_suffix + sep		) * [1]



					def lowerCamelCase_	(		self							:	Optional[int]     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	Optional[str] = None		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self._tokenizer.model.save(lowerCamelCase_     ,		name=lowerCamelCase_		)
							return tuple(lowerCamelCase_		)










'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase         =   logging.get_logger(__name__)

__UpperCAmelCase         =   {
    """uw-madison/yoso-4096""": """https://huggingface.co/uw-madison/yoso-4096/resolve/main/config.json""",
    # See all YOSO models at https://huggingface.co/models?filter=yoso
}




class       UpperCamelCase__			(			lowercase_	):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__						=					'''yoso'''


					def __init__(		self							:	str     ,		lowerCamelCase_							:	Tuple=5_02_65     ,		lowerCamelCase_							:	int=7_68     ,		lowerCamelCase_							:	Optional[Any]=12     ,		lowerCamelCase_							:	int=12     ,		lowerCamelCase_							:	Optional[Any]=30_72     ,		lowerCamelCase_							:	str="gelu"     ,		lowerCamelCase_							:	Union[str, Any]=0.1     ,		lowerCamelCase_							:	int=0.1     ,		lowerCamelCase_							:	Optional[int]=40_96     ,		lowerCamelCase_							:	str=1     ,		lowerCamelCase_							:	str=0.02     ,		lowerCamelCase_							:	Dict=1e-12     ,		lowerCamelCase_							:	Tuple="absolute"     ,		lowerCamelCase_							:	Any=True     ,		lowerCamelCase_							:	Dict=9     ,		lowerCamelCase_							:	Any=64     ,		lowerCamelCase_							:	Union[str, Any]=None     ,		lowerCamelCase_							:	List[str]=True     ,		lowerCamelCase_							:	Dict=True     ,		lowerCamelCase_							:	int=1     ,		lowerCamelCase_							:	Tuple=0     ,		lowerCamelCase_							:	Dict=2     ,		**lowerCamelCase_							:	Tuple     ,		):

							'''simple docstring'''



							super().__init__(pad_token_id=lowerCamelCase_     ,		bos_token_id=lowerCamelCase_     ,		eos_token_id=lowerCamelCase_     ,		**lowerCamelCase_		)

							SCREAMING_SNAKE_CASE				:							int						=	vocab_size
							SCREAMING_SNAKE_CASE				:							List[Any]						=	max_position_embeddings
							SCREAMING_SNAKE_CASE				:							Tuple						=	hidden_size
							SCREAMING_SNAKE_CASE				:							Dict						=	num_hidden_layers
							SCREAMING_SNAKE_CASE				:							Any						=	num_attention_heads
							SCREAMING_SNAKE_CASE				:							int						=	intermediate_size
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	hidden_act
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	hidden_dropout_prob
							SCREAMING_SNAKE_CASE				:							List[Any]						=	attention_probs_dropout_prob
							SCREAMING_SNAKE_CASE				:							str						=	initializer_range
							SCREAMING_SNAKE_CASE				:							List[str]						=	type_vocab_size
							SCREAMING_SNAKE_CASE				:							List[str]						=	layer_norm_eps
							SCREAMING_SNAKE_CASE				:							Any						=	position_embedding_type
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	use_expectation
							SCREAMING_SNAKE_CASE				:							int						=	hash_code_len
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	num_hash
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	conv_window
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	use_fast_hash
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	lsh_backward



