
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowercase__							:							int													=     logging.get_logger(__name__)

lowercase__							:							List[str]													=     {"vocab_file": "vocab.txt"}

lowercase__							:							Any													=     {
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

lowercase__							:							int													=     {
    "tau/splinter-base": 5_1_2,
    "tau/splinter-base-qass": 5_1_2,
    "tau/splinter-large": 5_1_2,
    "tau/splinter-large-qass": 5_1_2,
}

lowercase__							:							List[str]													=     {
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							_snake_case       						=					VOCAB_FILES_NAMES
							_snake_case       						=					PRETRAINED_VOCAB_FILES_MAP
							_snake_case       						=					PRETRAINED_INIT_CONFIGURATION
							_snake_case       						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_snake_case       						=					SplinterTokenizer



							def __init__(					self       ,		SCREAMING_SNAKE_CASE_=None       ,		SCREAMING_SNAKE_CASE_=None       ,		SCREAMING_SNAKE_CASE_=True       ,		SCREAMING_SNAKE_CASE_="[UNK]"       ,		SCREAMING_SNAKE_CASE_="[SEP]"       ,		SCREAMING_SNAKE_CASE_="[PAD]"       ,		SCREAMING_SNAKE_CASE_="[CLS]"       ,		SCREAMING_SNAKE_CASE_="[MASK]"       ,		SCREAMING_SNAKE_CASE_="[QUESTION]"       ,		SCREAMING_SNAKE_CASE_=True       ,		SCREAMING_SNAKE_CASE_=None       ,		**SCREAMING_SNAKE_CASE_       ,		)->				List[str]:





													'''simple docstring'''






													super().__init__(
													    SCREAMING_SNAKE_CASE_       ,		tokenizer_file=SCREAMING_SNAKE_CASE_       ,		do_lower_case=SCREAMING_SNAKE_CASE_       ,		unk_token=SCREAMING_SNAKE_CASE_       ,		sep_token=SCREAMING_SNAKE_CASE_       ,		pad_token=SCREAMING_SNAKE_CASE_       ,		cls_token=SCREAMING_SNAKE_CASE_       ,		mask_token=SCREAMING_SNAKE_CASE_       ,		tokenize_chinese_chars=SCREAMING_SNAKE_CASE_       ,		strip_accents=SCREAMING_SNAKE_CASE_       ,		additional_special_tokens=(question_token,)       ,		**SCREAMING_SNAKE_CASE_       ,		)

													__UpperCamelCase       					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
													if (
													    pre_tok_state.get('''lowercase'''       ,		SCREAMING_SNAKE_CASE_     ) != do_lower_case
													    or pre_tok_state.get('''strip_accents'''       ,		SCREAMING_SNAKE_CASE_     ) != strip_accents
													):
																			__UpperCamelCase       					=     getattr(SCREAMING_SNAKE_CASE_       ,		pre_tok_state.pop('''type'''     )     )
																			__UpperCamelCase       					=     do_lower_case
																			__UpperCamelCase       					=     strip_accents
																			__UpperCamelCase       					=     pre_tok_class(**SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     do_lower_case



							@property
							def       A__						(					self     )->				Tuple:





													'''simple docstring'''






													return self.convert_tokens_to_ids(self.question_token     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				List[int]:





													'''simple docstring'''






													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

													__UpperCamelCase       					=     [self.cls_token_id]
													__UpperCamelCase       					=     [self.sep_token_id]
													__UpperCamelCase       					=     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''     )]
													if self.padding_side == "right":
																			# Input is question-then-context
																			return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
													else:
																			# Input is context-then-question
																			return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				List[int]:





													'''simple docstring'''






													__UpperCamelCase       					=     [self.sep_token_id]
													__UpperCamelCase       					=     [self.cls_token_id]
													__UpperCamelCase       					=     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''     )]
													if token_ids_a is None:
																			return len(cls + token_ids_a + sep     ) * [0]

													if self.padding_side == "right":
																			# Input is question-then-context
																			return len(cls + token_ids_a + question_suffix + sep     ) * [0] + len(token_ids_a + sep     ) * [1]
													else:
																			# Input is context-then-question
																			return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + question_suffix + sep     ) * [1]



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				Tuple[str]:





													'''simple docstring'''






													__UpperCamelCase       					=     self._tokenizer.model.save(SCREAMING_SNAKE_CASE_       ,		name=SCREAMING_SNAKE_CASE_     )
													return tuple(SCREAMING_SNAKE_CASE_     )






import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							_snake_case       						=					CustomConfig



							def __init__(					self       ,		SCREAMING_SNAKE_CASE_     )->				Tuple:





													'''simple docstring'''






													super().__init__(SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     torch.nn.Linear(config.hidden_size       ,		config.hidden_size     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				List[str]:





													'''simple docstring'''






													return self.linear(SCREAMING_SNAKE_CASE_     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				Any:





													'''simple docstring'''






													pass






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							_snake_case       						=					NoSuperInitConfig



							def __init__(					self       ,		SCREAMING_SNAKE_CASE_     )->				Tuple:





													'''simple docstring'''






													super().__init__(SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     torch.nn.Linear(config.attribute       ,		config.attribute     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				List[str]:





													'''simple docstring'''






													return self.linear(SCREAMING_SNAKE_CASE_     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				Union[str, Any]:





													'''simple docstring'''






													pass




