


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        FunnelBaseModel,
        FunnelForMaskedLM,
        FunnelForMultipleChoice,
        FunnelForPreTraining,
        FunnelForQuestionAnswering,
        FunnelForSequenceClassification,
        FunnelForTokenClassification,
        FunnelModel,
    )







class 					lowerCAmelCase__	:




     '''simple docstring'''


     def __init__(						self			,						__lowerCamelCase			,						__lowerCamelCase=1_3			,						__lowerCamelCase=7			,						__lowerCamelCase=True			,						__lowerCamelCase=True			,						__lowerCamelCase=True			,						__lowerCamelCase=True			,						__lowerCamelCase=9_9			,						__lowerCamelCase=[1, 1, 2]			,						__lowerCamelCase=1			,						__lowerCamelCase=3_2			,						__lowerCamelCase=4			,						__lowerCamelCase=8			,						__lowerCamelCase=3_7			,						__lowerCamelCase="gelu_new"			,						__lowerCamelCase=0.1			,						__lowerCamelCase=0.1			,						__lowerCamelCase=0.0			,						__lowerCamelCase=5_1_2			,						__lowerCamelCase=3			,						__lowerCamelCase=0.0_2			,						__lowerCamelCase=3			,						__lowerCamelCase=4			,						__lowerCamelCase=None			,						__lowerCamelCase=False			,						)      ->    Any:
            _A				:	Tuple       =			parent
            _A				:	Dict       =			batch_size
            _A				:	List[Any]       =			seq_length
            _A				:	int       =			is_training
            _A				:	Union[str, Any]       =			use_input_mask
            _A				:	Dict       =			use_token_type_ids
            _A				:	str       =			use_labels
            _A				:	Dict       =			vocab_size
            _A				:	Optional[int]       =			block_sizes
            _A				:	Tuple       =			num_decoder_layers
            _A				:	Optional[int]       =			d_model
            _A				:	Union[str, Any]       =			n_head
            _A				:	int       =			d_head
            _A				:	str       =			d_inner
            _A				:	Optional[int]       =			hidden_act
            _A				:	List[str]       =			hidden_dropout
            _A				:	Dict       =			attention_dropout
            _A				:	List[str]       =			activation_dropout
            _A				:	List[str]       =			max_position_embeddings
            _A				:	Tuple       =			type_vocab_size
            _A				:	str       =			2
            _A				:	Optional[Any]       =			num_labels
            _A				:	Union[str, Any]       =			num_choices
            _A				:	Optional[int]       =			scope
            _A				:	Union[str, Any]       =			initializer_std

            # Used in the tests to check the size of the first attention layer
            _A				:	Dict       =			n_head
            # Used in the tests to check the size of the first hidden state
            _A				:	List[Any]       =			self.d_model
            # Used in the tests to check the number of output hidden states/attentions
            _A				:	List[Any]       =			sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
            # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
            # the last hidden state of the first block (which is the first hidden state of the decoder).
            if not base:
                   _A				:	Any       =			self.num_hidden_layers + 2


     def 			_lowerCamelCase			(						self)      ->    Tuple:
            _A				:	int       =			ids_tensor([self.batch_size, self.seq_length]			,						self.vocab_size)

            _A				:	Any       =			None
            if self.use_input_mask:
                   _A				:	List[Any]       =			ids_tensor([self.batch_size, self.seq_length]			,						vocab_size=2)

            _A				:	Optional[int]       =			None
            if self.use_token_type_ids:
                   _A				:	Optional[Any]       =			ids_tensor([self.batch_size, self.seq_length]			,						self.type_vocab_size)

            _A				:	List[str]       =			None
            _A				:	Dict       =			None
            _A				:	List[str]       =			None
            if self.use_labels:
                   _A				:	Tuple       =			ids_tensor([self.batch_size]			,						self.type_sequence_label_size)
                   _A				:	str       =			ids_tensor([self.batch_size, self.seq_length]			,						self.num_labels)
                   _A				:	Any       =			ids_tensor([self.batch_size]			,						self.num_choices)
                   _A				:	List[Any]       =			ids_tensor([self.batch_size, self.seq_length]			,						1)

            _A				:	int       =			self.get_config()

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                fake_token_labels,
            )


     def 			_lowerCamelCase			(						self)      ->    int:
            return FunnelConfig(
                vocab_size=self.vocab_size			,						block_sizes=self.block_sizes			,						num_decoder_layers=self.num_decoder_layers			,						d_model=self.d_model			,						n_head=self.n_head			,						d_head=self.d_head			,						d_inner=self.d_inner			,						hidden_act=self.hidden_act			,						hidden_dropout=self.hidden_dropout			,						attention_dropout=self.attention_dropout			,						activation_dropout=self.activation_dropout			,						max_position_embeddings=self.max_position_embeddings			,						type_vocab_size=self.type_vocab_size			,						initializer_std=self.initializer_std			,						)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    int:
            _A				:	List[Any]       =			FunnelModel(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	List[Any]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase)
            _A				:	Union[str, Any]       =			model(__lowerCamelCase			,						token_type_ids=__lowerCamelCase)
            _A				:	str       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, self.seq_length, self.d_model))

            _A				:	Optional[Any]       =			False
            _A				:	Dict       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, self.seq_length, self.d_model))

            _A				:	Any       =			False
            _A				:	Optional[int]       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, self.seq_length, self.d_model))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    Tuple:
            _A				:	Optional[Any]       =			FunnelBaseModel(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	Optional[Any]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase)
            _A				:	Optional[int]       =			model(__lowerCamelCase			,						token_type_ids=__lowerCamelCase)
            _A				:	Tuple       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, 2, self.d_model))

            _A				:	List[Any]       =			False
            _A				:	int       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, 3, self.d_model))

            _A				:	Dict       =			False
            _A				:	int       =			model(__lowerCamelCase)
            self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, 2, self.d_model))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    str:
            _A				:	Tuple       =			self.num_labels
            _A				:	Optional[Any]       =			FunnelForPreTraining(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	List[Any]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						labels=__lowerCamelCase)
            self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.seq_length))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    List[Any]:
            _A				:	str       =			FunnelForMaskedLM(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	int       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						labels=__lowerCamelCase)
            self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.seq_length, self.vocab_size))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    Tuple:
            _A				:	List[str]       =			self.num_labels
            _A				:	List[Any]       =			FunnelForSequenceClassification(__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	Optional[Any]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						labels=__lowerCamelCase)
            self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.num_labels))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    Dict:
            _A				:	List[str]       =			self.num_choices
            _A				:	Dict       =			FunnelForMultipleChoice(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	Dict       =			input_ids.unsqueeze(1).expand(-1			,						self.num_choices			,						-1).contiguous()
            _A				:	Tuple       =			token_type_ids.unsqueeze(1).expand(-1			,						self.num_choices			,						-1).contiguous()
            _A				:	str       =			input_mask.unsqueeze(1).expand(-1			,						self.num_choices			,						-1).contiguous()
            _A				:	Any       =			model(
                __lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						labels=__lowerCamelCase			,						)
            self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.num_choices))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    Dict:
            _A				:	Optional[int]       =			self.num_labels
            _A				:	Optional[int]       =			FunnelForTokenClassification(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	Optional[Any]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						labels=__lowerCamelCase)
            self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.seq_length, self.num_labels))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase			,						)      ->    List[Any]:
            _A				:	Optional[Any]       =			FunnelForQuestionAnswering(config=__lowerCamelCase)
            model.to(__lowerCamelCase)
            model.eval()
            _A				:	Union[str, Any]       =			model(
                __lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase			,						start_positions=__lowerCamelCase			,						end_positions=__lowerCamelCase			,						)
            self.parent.assertEqual(result.start_logits.shape			,						(self.batch_size, self.seq_length))
            self.parent.assertEqual(result.end_logits.shape			,						(self.batch_size, self.seq_length))


     def 			_lowerCamelCase			(						self)      ->    Optional[int]:
            _A				:	Optional[int]       =			self.prepare_config_and_inputs()
            (
                (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      (
                _A
            )   ,      
            )				:	int       =			config_and_inputs
            _A				:	Optional[int]       =			{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
            return config, inputs_dict







@require_torch
class 					lowerCAmelCase__	(    a					,   a					,   unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			False
     __SCREAMING_SNAKE_CASE        =			False
     __SCREAMING_SNAKE_CASE        =			(
         (
             FunnelModel,
             FunnelForMaskedLM,
             FunnelForPreTraining,
             FunnelForQuestionAnswering,
             FunnelForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     __SCREAMING_SNAKE_CASE        =			(
         {
             "feature-extraction": (FunnelBaseModel, FunnelModel),
             "fill-mask": FunnelForMaskedLM,
             "question-answering": FunnelForQuestionAnswering,
             "text-classification": FunnelForSequenceClassification,
             "token-classification": FunnelForTokenClassification,
             "zero-shot": FunnelForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase=False)      ->    Tuple:
            _A				:	Dict       =			super()._prepare_for_class(__lowerCamelCase			,						__lowerCamelCase			,						return_labels=__lowerCamelCase)

            if return_labels:
                   if model_class in get_values(__lowerCamelCase):
                          _A				:	str       =			torch.zeros(
                              (self.model_tester.batch_size, self.model_tester.seq_length)			,						dtype=torch.long			,						device=__lowerCamelCase)
            return inputs_dict


     def 			_lowerCamelCase			(						self)      ->    List[Any]:
            _A				:	Tuple       =			FunnelModelTester(self)
            _A				:	Tuple       =			ConfigTester(self			,						config_class=__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    List[str]:
            self.config_tester.run_common_tests()


     def 			_lowerCamelCase			(						self)      ->    int:
            _A				:	Tuple       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    Any:
            _A				:	Dict       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_pretraining(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    Union[str, Any]:
            _A				:	List[Any]       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    Optional[int]:
            _A				:	str       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    List[str]:
            _A				:	Union[str, Any]       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    str:
            if hasattr(__lowerCamelCase			,						"weight") and module.weight is not None:
                   module.weight.data.fill_(3)
            if hasattr(__lowerCamelCase			,						"bias") and module.bias is not None:
                   module.bias.data.fill_(3)

            for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                   if hasattr(__lowerCamelCase			,						__lowerCamelCase) and getattr(__lowerCamelCase			,						__lowerCamelCase) is not None:
                          _A				:	Tuple       =			getattr(__lowerCamelCase			,						__lowerCamelCase)
                          weight.data.fill_(3)







@require_torch
class 					lowerCAmelCase__	(    a					,   unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			False
     __SCREAMING_SNAKE_CASE        =			False
     __SCREAMING_SNAKE_CASE        =			(
         (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
     )


     def 			_lowerCamelCase			(						self)      ->    Any:
            _A				:	Union[str, Any]       =			FunnelModelTester(self			,						base=__lowerCamelCase)
            _A				:	Dict       =			ConfigTester(self			,						config_class=__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    Optional[Any]:
            self.config_tester.run_common_tests()


     def 			_lowerCamelCase			(						self)      ->    str:
            _A				:	Optional[int]       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_base_model(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    str:
            _A				:	Optional[int]       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    Optional[int]:
            _A				:	Any       =			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*__lowerCamelCase)


     def 			_lowerCamelCase			(						self)      ->    List[Any]:
            _A   ,      _A				:	Union[str, Any]       =			self.model_tester.prepare_config_and_inputs_for_common()
            _A				:	List[Any]       =			True

            for model_class in self.all_model_classes:
                   if model_class.__name__ == "FunnelBaseModel":
                          continue
                   _A				:	Optional[Any]       =			model_class(__lowerCamelCase)
                   model.to(__lowerCamelCase)
                   model.train()
                   _A				:	str       =			self._prepare_for_class(__lowerCamelCase			,						__lowerCamelCase			,						return_labels=__lowerCamelCase)
                   _A				:	str       =			model(**__lowerCamelCase).loss
                   loss.backward()


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    Optional[Any]:
            if hasattr(__lowerCamelCase			,						"weight") and module.weight is not None:
                   module.weight.data.fill_(3)
            if hasattr(__lowerCamelCase			,						"bias") and module.bias is not None:
                   module.bias.data.fill_(3)

            for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                   if hasattr(__lowerCamelCase			,						__lowerCamelCase) and getattr(__lowerCamelCase			,						__lowerCamelCase) is not None:
                          _A				:	int       =			getattr(__lowerCamelCase			,						__lowerCamelCase)
                          weight.data.fill_(3)







@require_torch
@require_sentencepiece
@require_tokenizers
class 					lowerCAmelCase__	(    unittest.TestCase):




     '''simple docstring'''


     def 			_lowerCamelCase			(						self)      ->    List[str]:
            _A				:	str       =			1_3
            _A				:	Union[str, Any]       =			7
            _A				:	Optional[Any]       =			torch.arange(0			,						batch_size * sequence_length).long().reshape(__lowerCamelCase			,						__lowerCamelCase)
            _A				:	Any       =			[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
            _A				:	Tuple       =			torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths])

            _A				:	List[str]       =			FunnelModel.from_pretrained("sgugger/funnel-random-tiny")
            _A				:	Optional[int]       =			model(__lowerCamelCase			,						token_type_ids=__lowerCamelCase)[0].abs()

            _A				:	str       =			torch.tensor(2_3_4_4.8_3_5_2)
            _A				:	str       =			torch.tensor(0.8_0_5_2)
            self.assertTrue(torch.allclose(output.sum()			,						__lowerCamelCase			,						atol=1e-4))
            self.assertTrue(torch.allclose(output.mean()			,						__lowerCamelCase			,						atol=1e-4))

            _A				:	List[Any]       =			torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]])
            _A				:	List[str]       =			model(__lowerCamelCase			,						attention_mask=__lowerCamelCase			,						token_type_ids=__lowerCamelCase)[0].abs()

            _A				:	List[str]       =			torch.tensor(2_3_4_3.8_4_2_5)
            _A				:	Union[str, Any]       =			torch.tensor(0.8_0_4_9)
            self.assertTrue(torch.allclose(output.sum()			,						__lowerCamelCase			,						atol=1e-4))
            self.assertTrue(torch.allclose(output.mean()			,						__lowerCamelCase			,						atol=1e-4))


     @slow
     def 			_lowerCamelCase			(						self)      ->    List[Any]:
            _A				:	Optional[Any]       =			FunnelTokenizer.from_pretrained("huggingface/funnel-small")
            _A				:	Tuple       =			FunnelModel.from_pretrained("huggingface/funnel-small")
            _A				:	List[str]       =			tokenizer("Hello! I am the Funnel Transformer model."			,						return_tensors="pt")
            _A				:	Any       =			model(**__lowerCamelCase)[0]

            _A				:	str       =			torch.tensor(2_3_5.7_2_4_6)
            _A				:	str       =			torch.tensor(0.0_2_5_6)
            self.assertTrue(torch.allclose(output.sum()			,						__lowerCamelCase			,						atol=1e-4))
            self.assertTrue(torch.allclose(output.mean()			,						__lowerCamelCase			,						atol=1e-4))




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase__						=			logging.get_logger(__name__)

lowerCAmelCase__						=			{'vocab_file': 'vocab.txt'}

lowerCAmelCase__						=			{
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

lowerCAmelCase__						=			{
    'tau/splinter-base': 5_12,
    'tau/splinter-base-qass': 5_12,
    'tau/splinter-large': 5_12,
    'tau/splinter-large-qass': 5_12,
}

lowerCAmelCase__						=			{
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}







class 					lowerCAmelCase__	(    a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_INIT_CONFIGURATION
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE        =			SplinterTokenizer


     def __init__(						self			,						__lowerCamelCase=None			,						__lowerCamelCase=None			,						__lowerCamelCase=True			,						__lowerCamelCase="[UNK]"			,						__lowerCamelCase="[SEP]"			,						__lowerCamelCase="[PAD]"			,						__lowerCamelCase="[CLS]"			,						__lowerCamelCase="[MASK]"			,						__lowerCamelCase="[QUESTION]"			,						__lowerCamelCase=True			,						__lowerCamelCase=None			,						**__lowerCamelCase			,						)      ->    Union[str, Any]:
            super().__init__(
                __lowerCamelCase			,						tokenizer_file=__lowerCamelCase			,						do_lower_case=__lowerCamelCase			,						unk_token=__lowerCamelCase			,						sep_token=__lowerCamelCase			,						pad_token=__lowerCamelCase			,						cls_token=__lowerCamelCase			,						mask_token=__lowerCamelCase			,						tokenize_chinese_chars=__lowerCamelCase			,						strip_accents=__lowerCamelCase			,						additional_special_tokens=(question_token,)			,						**__lowerCamelCase			,						)

            _A				:	Tuple       =			json.loads(self.backend_tokenizer.normalizer.__getstate__())
            if (
                pre_tok_state.get("lowercase"			,						__lowerCamelCase) != do_lower_case
                or pre_tok_state.get("strip_accents"			,						__lowerCamelCase) != strip_accents
            ):
                   _A				:	Dict       =			getattr(__lowerCamelCase			,						pre_tok_state.pop("type"))
                   _A				:	List[str]       =			do_lower_case
                   _A				:	Any       =			strip_accents
                   _A				:	int       =			pre_tok_class(**__lowerCamelCase)

            _A				:	int       =			do_lower_case


     @property
     def 			_lowerCamelCase			(						self)      ->    str:
            return self.convert_tokens_to_ids(self.question_token)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    List[int]:
            if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

            _A				:	Tuple       =			[self.cls_token_id]
            _A				:	str       =			[self.sep_token_id]
            _A				:	int       =			[self.question_token_id] + [self.convert_tokens_to_ids(".")]
            if self.padding_side == "right":
                   # Input is question-then-context
                   return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
            else:
                   # Input is context-then-question
                   return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    List[int]:
            _A				:	str       =			[self.sep_token_id]
            _A				:	int       =			[self.cls_token_id]
            _A				:	Union[str, Any]       =			[self.question_token_id] + [self.convert_tokens_to_ids(".")]
            if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]

            if self.padding_side == "right":
                   # Input is question-then-context
                   return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
            else:
                   # Input is context-then-question
                   return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    Tuple[str]:
            _A				:	Tuple       =			self._tokenizer.model.save(__lowerCamelCase			,						name=__lowerCamelCase)
            return tuple(__lowerCamelCase)
