




"""simple docstring"""







class  SCREAMING_SNAKE_CASE__   :






						class  SCREAMING_SNAKE_CASE__   :
												_a        =  '_prev', '_data', '_next'


												def __init__(							self			:     Dict  ,  lowerCAmelCase			:     Tuple  ,  lowerCAmelCase			:     Optional[int]  ,  lowerCAmelCase			:     List[str]   ):
																lowerCAmelCase		=			link_p
																lowerCAmelCase		=			element
																lowerCAmelCase		=			link_n




												def 					__lowercase	(							self			:     Dict   ):
																return (
																    f''' Prev -> {self._prev is not None}, Next -> {self._next is not None}'''
																)





						def __init__(							self			:     Any   ):
										lowerCAmelCase		=			self._Node(lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										lowerCAmelCase		=			self._Node(lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										lowerCAmelCase		=			self._trailer
										lowerCAmelCase		=			self._header
										lowerCAmelCase		=			0


						def __len__(							self			:     Union[str, Any]   ):
										return self._size


						def 					__lowercase	(							self			:     int   ):
										return self.__len__() == 0


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     Dict  ,  lowerCAmelCase			:     Union[str, Any]  ,  lowerCAmelCase			:     Optional[int]   ):
										# Create new_node by setting it's prev.link -> header
										# setting it's next.link -> trailer
										lowerCAmelCase		=			self._Node(lowerCAmelCase  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										lowerCAmelCase		=			new_node
										lowerCAmelCase		=			new_node
										self._size += 1
										return self




						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     Optional[Any]   ):
										lowerCAmelCase		=			node._prev
										lowerCAmelCase		=			node._next

										lowerCAmelCase		=			successor
										lowerCAmelCase		=			predecessor
										self._size -= 1
										lowerCAmelCase		=			node._data
										lowerCAmelCase		=			lowerCAmelCase		=			lowerCAmelCase		=			None
										del node
										return temp






class  SCREAMING_SNAKE_CASE__   (   _DoublyLinkedBase   ):


						def 					__lowercase	(							self			:     Dict   ):
										if self.is_empty():
														raise Exception("""List is empty"""   )
										return self._header._next._data


						def 					__lowercase	(							self			:     str   ):
										if self.is_empty():
														raise Exception("""List is empty"""   )
										return self._trailer._prev._data


						def 					__lowercase	(							self			:     List[Any]  ,  lowerCAmelCase			:     Optional[Any]   ):
										return self._insert(self._header  ,  lowerCAmelCase  ,  self._header._next   )


						def 					__lowercase	(							self			:     Dict  ,  lowerCAmelCase			:     Tuple   ):
										return self._insert(self._trailer._prev  ,  lowerCAmelCase  ,  self._trailer   )


						def 					__lowercase	(							self			:     Optional[int]   ):
										if self.is_empty():
														raise IndexError("""remove_first from empty list"""   )
										return self._delete(self._header._next   )




						def 					__lowercase	(							self			:     Any   ):
										if self.is_empty():
														raise IndexError("""remove_first from empty list"""   )
										return self._delete(self._trailer._prev   )











"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


a								=							logging.get_logger(__name__)

a								=							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a								=							{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

a								=							{
    'allenai/longformer-base-4096': 4_0_9_6,
    'allenai/longformer-large-4096': 4_0_9_6,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_0_9_6,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_0_9_6,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_0_9_6,
}






class  SCREAMING_SNAKE_CASE__   (   _a   ):
						_a        =  VOCAB_FILES_NAMES
						_a        =  PRETRAINED_VOCAB_FILES_MAP
						_a        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a        =  ['input_ids', 'attention_mask']
						_a        =  LongformerTokenizer


						def __init__(							self			:     Optional[Any]  ,  lowerCAmelCase			:     Tuple=None  ,  lowerCAmelCase			:     Any=None  ,  lowerCAmelCase			:     List[Any]=None  ,  lowerCAmelCase			:     Optional[Any]="replace"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     int="</s>"  ,  lowerCAmelCase			:     Any="</s>"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     List[str]="<unk>"  ,  lowerCAmelCase			:     Optional[Any]="<pad>"  ,  lowerCAmelCase			:     List[str]="<mask>"  ,  lowerCAmelCase			:     Tuple=False  ,  lowerCAmelCase			:     List[str]=True  ,  **lowerCAmelCase			:     Optional[int]  ,  ):
										super().__init__(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  tokenizer_file=lowerCAmelCase  ,  errors=lowerCAmelCase  ,  bos_token=lowerCAmelCase  ,  eos_token=lowerCAmelCase  ,  sep_token=lowerCAmelCase  ,  cls_token=lowerCAmelCase  ,  unk_token=lowerCAmelCase  ,  pad_token=lowerCAmelCase  ,  mask_token=lowerCAmelCase  ,  add_prefix_space=lowerCAmelCase  ,  trim_offsets=lowerCAmelCase  ,  **lowerCAmelCase  ,  )

										lowerCAmelCase		=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
										if pre_tok_state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
														lowerCAmelCase		=			getattr(lowerCAmelCase  ,  pre_tok_state.pop("""type"""   )   )
														lowerCAmelCase		=			add_prefix_space
														lowerCAmelCase		=			pre_tok_class(**lowerCAmelCase   )

										lowerCAmelCase		=			add_prefix_space

										lowerCAmelCase		=			"""post_processor"""
										lowerCAmelCase		=			getattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										if tokenizer_component_instance:
														lowerCAmelCase		=			json.loads(tokenizer_component_instance.__getstate__()   )

														# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
														if "sep" in state:
																		lowerCAmelCase		=			tuple(state["""sep"""]   )
														if "cls" in state:
																		lowerCAmelCase		=			tuple(state["""cls"""]   )

														lowerCAmelCase		=			False

														if state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
																		lowerCAmelCase		=			add_prefix_space
																		lowerCAmelCase		=			True

														if state.get("""trim_offsets"""  ,  lowerCAmelCase   ) != trim_offsets:
																		lowerCAmelCase		=			trim_offsets
																		lowerCAmelCase		=			True

														if changes_to_apply:
																		lowerCAmelCase		=			getattr(lowerCAmelCase  ,  state.pop("""type"""   )   )
																		lowerCAmelCase		=			component_class(**lowerCAmelCase   )
																		setattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )


						@property
						def 					__lowercase	(							self			:     List[Any]   ):
										if self._mask_token is None:
														if self.verbose:
																		logger.error("""Using mask_token, but it is not set yet."""   )
														return None
										return str(self._mask_token   )


						@mask_token.setter
						def 					__lowercase	(							self			:     List[Any]  ,  lowerCAmelCase			:     Dict   ):
										lowerCAmelCase		=			AddedToken(lowerCAmelCase  ,  lstrip=lowerCAmelCase  ,  rstrip=lowerCAmelCase   ) if isinstance(lowerCAmelCase  ,  lowerCAmelCase   ) else value
										lowerCAmelCase		=			value


						def 					__lowercase	(							self			:     Tuple  ,  *lowerCAmelCase			:     Optional[int]  ,  **lowerCAmelCase			:     Optional[Any]   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     str  ,  *lowerCAmelCase			:     str  ,  **lowerCAmelCase			:     str   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     str  ,  lowerCAmelCase			:     Optional[str] = None   ):
										lowerCAmelCase		=			self._tokenizer.model.save(lowerCAmelCase  ,  name=lowerCAmelCase   )
										return tuple(lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[Any]  ,  lowerCAmelCase			:     List[str]  ,  lowerCAmelCase			:     Optional[int]=None   ):
										lowerCAmelCase		=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
														return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     List[int]  ,  lowerCAmelCase			:     Optional[List[int]] = None   ):
										lowerCAmelCase		=			[self.sep_token_id]
										lowerCAmelCase		=			[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





