




"""simple docstring"""







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a								=							logging.get_logger(__name__)

a								=							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a								=							{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

a								=							{
    'openai-gpt': 5_1_2,
}





def 							lowercase   (snake_case__				:	int	)       ->					Any:


				'''simple docstring'''






				lowerCAmelCase		=			text.strip()
				if not text:
								return []
				lowerCAmelCase		=			text.split()
				return tokens






class  SCREAMING_SNAKE_CASE__   (   lowerCAmelCase__   ):


						def __init__(							self			:     Optional[Any]  ,  lowerCAmelCase			:     Optional[int]=True  ,  lowerCAmelCase			:     Dict=None  ,  lowerCAmelCase			:     str=True  ,  lowerCAmelCase			:     List[str]=None  ,  lowerCAmelCase			:     Optional[Any]=True  ,  ):




										if never_split is None:
														lowerCAmelCase		=			[]
										lowerCAmelCase		=			do_lower_case
										lowerCAmelCase		=			set(_A   )
										lowerCAmelCase		=			tokenize_chinese_chars
										lowerCAmelCase		=			strip_accents
										lowerCAmelCase		=			do_split_on_punc


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     int  ,  lowerCAmelCase			:     List[str]=None   ):
										lowerCAmelCase		=			self.never_split.union(set(_A   )   ) if never_split else self.never_split
										lowerCAmelCase		=			self._clean_text(_A   )

										# This was added on November 1st, 2018 for the multilingual and Chinese
										# models. This is also applied to the English models now, but it doesn't
										# matter since the English models were not trained on any Chinese data
										# and generally don't have any Chinese data in them (there are Chinese
										# characters in the vocabulary because Wikipedia does have some Chinese
										# words in the English Wikipedia.).
										if self.tokenize_chinese_chars:
														lowerCAmelCase		=			self._tokenize_chinese_chars(_A   )
										# prevents treating the same character with different unicode codepoints as different characters
										lowerCAmelCase		=			unicodedata.normalize("""NFC"""  ,  _A   )
										lowerCAmelCase		=			whitespace_tokenize(_A   )
										lowerCAmelCase		=			[]
										for token in orig_tokens:
														if token not in never_split:
																		if self.do_lower_case:
																						lowerCAmelCase		=			token.lower()
																						if self.strip_accents is not False:
																										lowerCAmelCase		=			self._run_strip_accents(_A   )
																		elif self.strip_accents:
																						lowerCAmelCase		=			self._run_strip_accents(_A   )
														split_tokens.extend(self._run_split_on_punc(_A  ,  _A   )   )

										lowerCAmelCase		=			whitespace_tokenize(""" """.join(_A   )   )
										return output_tokens


						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     List[Any]   ):
										lowerCAmelCase		=			unicodedata.normalize("""NFD"""  ,  _A   )
										lowerCAmelCase		=			[]
										for char in text:
														lowerCAmelCase		=			unicodedata.category(_A   )
														if cat == "Mn":
																		continue
														output.append(_A   )
										return "".join(_A   )


						def 					__lowercase	(							self			:     Dict  ,  lowerCAmelCase			:     Union[str, Any]  ,  lowerCAmelCase			:     Union[str, Any]=None   ):




										if not self.do_split_on_punc or (never_split is not None and text in never_split):
														return [text]
										lowerCAmelCase		=			list(_A   )
										lowerCAmelCase		=			0
										lowerCAmelCase		=			True
										lowerCAmelCase		=			[]
										while i < len(_A   ):
														lowerCAmelCase		=			chars[i]
														if _is_punctuation(_A   ):
																		output.append([char]   )
																		lowerCAmelCase		=			True
														else:
																		if start_new_word:
																						output.append([]   )
																		lowerCAmelCase		=			False
																		output[-1].append(_A   )
														i += 1

										return ["".join(_A   ) for x in output]


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     Any   ):
										lowerCAmelCase		=			[]
										for char in text:
														lowerCAmelCase		=			ord(_A   )
														if self._is_chinese_char(_A   ):
																		output.append(""" """   )
																		output.append(_A   )
																		output.append(""" """   )
														else:
																		output.append(_A   )
										return "".join(_A   )


						def 					__lowercase	(							self			:     str  ,  lowerCAmelCase			:     List[str]   ):




										if (
										    (cp >= 0X4e_00 and cp <= 0X9f_ff)
										    or (cp >= 0X34_00 and cp <= 0X4d_bf)  #
										    or (cp >= 0X2_00_00 and cp <= 0X2_a6_df)  #
										    or (cp >= 0X2_a7_00 and cp <= 0X2_b7_3f)  #
										    or (cp >= 0X2_b7_40 and cp <= 0X2_b8_1f)  #
										    or (cp >= 0X2_b8_20 and cp <= 0X2_ce_af)  #
										    or (cp >= 0Xf9_00 and cp <= 0Xfa_ff)
										    or (cp >= 0X2_f8_00 and cp <= 0X2_fa_1f)  #
										):  #
														return True

										return False




						def 					__lowercase	(							self			:     int  ,  lowerCAmelCase			:     Optional[Any]   ):
										lowerCAmelCase		=			[]
										for char in text:
														lowerCAmelCase		=			ord(_A   )
														if cp == 0 or cp == 0Xff_fd or _is_control(_A   ):
																		continue
														if _is_whitespace(_A   ):
																		output.append(""" """   )
														else:
																		output.append(_A   )
										return "".join(_A   )





def 							lowercase   (snake_case__				:	Optional[int]	)       ->					List[str]:


				'''simple docstring'''






				lowerCAmelCase		=			set()
				lowerCAmelCase		=			word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)	)
								lowerCAmelCase		=			char
				return pairs





def 							lowercase   (snake_case__				:	int	)       ->					Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			text.replace("""—""" ,    """-"""	)
				lowerCAmelCase		=			text.replace("""–""" ,    """-"""	)
				lowerCAmelCase		=			text.replace("""―""" ,    """-"""	)
				lowerCAmelCase		=			text.replace("""…""" ,    """..."""	)
				lowerCAmelCase		=			text.replace("""´""" ,    """\'"""	)
				lowerCAmelCase		=			re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""" ,    R""" \1 """ ,    snake_case__	)
				lowerCAmelCase		=			re.sub(R"""\s*\n\s*""" ,    """ \n """ ,    snake_case__	)
				lowerCAmelCase		=			re.sub(R"""[^\S\n]+""" ,    """ """ ,    snake_case__	)
				return text.strip()






class  SCREAMING_SNAKE_CASE__   (   lowerCAmelCase__   ):
						_a        =  VOCAB_FILES_NAMES
						_a        =  PRETRAINED_VOCAB_FILES_MAP
						_a        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a        =  ['input_ids', 'attention_mask']


						def __init__(							self			:     Optional[Any]  ,  lowerCAmelCase			:     List[Any]  ,  lowerCAmelCase			:     List[str]  ,  lowerCAmelCase			:     Optional[int]="<unk>"  ,  **lowerCAmelCase			:     Optional[Any]   ):
										super().__init__(unk_token=_A  ,  **_A   )

										try:
														import ftfy
														from spacy.lang.en import English

														lowerCAmelCase		=			English()
														lowerCAmelCase		=			_nlp.tokenizer
														lowerCAmelCase		=			ftfy.fix_text
										except ImportError:
														logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""   )
														lowerCAmelCase		=			BasicTokenizer(do_lower_case=_A   )
														lowerCAmelCase		=			None

										with open(_A  ,  encoding="""utf-8"""   ) as vocab_handle:
														lowerCAmelCase		=			json.load(_A   )
										lowerCAmelCase		=			{v: k for k, v in self.encoder.items()}
										with open(_A  ,  encoding="""utf-8"""   ) as merges_handle:
														lowerCAmelCase		=			merges_handle.read().split("""\n"""   )[1:-1]
										lowerCAmelCase		=			[tuple(merge.split()   ) for merge in merges]
										lowerCAmelCase		=			dict(zip(_A  ,  range(len(_A   )   )   )   )
										lowerCAmelCase		=			{}


						@property
						def 					__lowercase	(							self			:     Tuple   ):
										return True


						@property
						def 					__lowercase	(							self			:     Dict   ):
										return len(self.encoder   )


						def 					__lowercase	(							self			:     Optional[Any]   ):
										return dict(self.encoder  ,  **self.added_tokens_encoder   )


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     List[Any]   ):
										lowerCAmelCase		=			tuple(token[:-1]   ) + (token[-1] + '''</w>''',)
										if token in self.cache:
														return self.cache[token]
										lowerCAmelCase		=			get_pairs(_A   )

										if not pairs:
														return token + "</w>"

										while True:
														lowerCAmelCase		=			min(_A  ,  key=lambda lowerCAmelCase   : self.bpe_ranks.get(_A  ,  float("""inf"""   )   )   )
														if bigram not in self.bpe_ranks:
																		break
														lowerCAmelCase		=			bigram
														lowerCAmelCase		=			[]
														lowerCAmelCase		=			0
														while i < len(_A   ):
																		try:
																						lowerCAmelCase		=			word.index(_A  ,  _A   )
																		except ValueError:
																						new_word.extend(word[i:]   )
																						break
																		else:
																						new_word.extend(word[i:j]   )
																						lowerCAmelCase		=			j

																		if word[i] == first and i < len(_A   ) - 1 and word[i + 1] == second:
																						new_word.append(first + second   )
																						i += 2
																		else:
																						new_word.append(word[i]   )
																						i += 1
														lowerCAmelCase		=			tuple(_A   )
														lowerCAmelCase		=			new_word
														if len(_A   ) == 1:
																		break
														else:
																		lowerCAmelCase		=			get_pairs(_A   )
										lowerCAmelCase		=			''' '''.join(_A   )
										if word == "\n  </w>":
														lowerCAmelCase		=			'''\n</w>'''
										lowerCAmelCase		=			word
										return word


						def 					__lowercase	(							self			:     Tuple  ,  lowerCAmelCase			:     List[str]   ):
										lowerCAmelCase		=			[]
										if self.fix_text is None:
														# Using BERT's BasicTokenizer
														lowerCAmelCase		=			self.nlp.tokenize(_A   )
														for token in text:
																		split_tokens.extend(list(self.bpe(_A   ).split(""" """   )   )   )
										else:
														# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
														lowerCAmelCase		=			self.nlp(text_standardize(self.fix_text(_A   )   )   )
														for token in text:
																		split_tokens.extend(list(self.bpe(token.text.lower()   ).split(""" """   )   )   )
										return split_tokens


						def 					__lowercase	(							self			:     int  ,  lowerCAmelCase			:     Tuple   ):
										return self.encoder.get(_A  ,  self.encoder.get(self.unk_token   )   )


						def 					__lowercase	(							self			:     Union[str, Any]  ,  lowerCAmelCase			:     Any   ):
										return self.decoder.get(_A  ,  self.unk_token   )


						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     str   ):
										lowerCAmelCase		=			''''''.join(_A   ).replace("""</w>"""  ,  """ """   ).strip()
										return out_string




						def 					__lowercase	(							self			:     List[Any]  ,  lowerCAmelCase			:     str  ,  lowerCAmelCase			:     Optional[str] = None   ):




										if not os.path.isdir(_A   ):
														logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
														return
										lowerCAmelCase		=			os.path.join(
										    _A  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
										lowerCAmelCase		=			os.path.join(
										    _A  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

										with open(_A  ,  """w"""  ,  encoding="""utf-8"""   ) as f:
														f.write(json.dumps(self.encoder  ,  indent=2  ,  sort_keys=_A  ,  ensure_ascii=_A   ) + """\n"""   )

										lowerCAmelCase		=			0
										with open(_A  ,  """w"""  ,  encoding="""utf-8"""   ) as writer:
														writer.write("""#version: 0.2\n"""   )
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,  key=lambda lowerCAmelCase   : kv[1]   ):
																		if index != token_index:
																						logger.warning(
																						    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																						    """ Please check that the tokenizer is not corrupted!"""   )
																						lowerCAmelCase		=			token_index
																		writer.write(""" """.join(_A   ) + """\n"""   )
																		index += 1

										return vocab_file, merge_file











"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


a								=							logging.get_logger(__name__)

a								=							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a								=							{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

a								=							{
    'allenai/longformer-base-4096': 4_0_9_6,
    'allenai/longformer-large-4096': 4_0_9_6,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_0_9_6,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_0_9_6,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_0_9_6,
}






class  SCREAMING_SNAKE_CASE__   (   _a   ):
						_a        =  VOCAB_FILES_NAMES
						_a        =  PRETRAINED_VOCAB_FILES_MAP
						_a        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a        =  ['input_ids', 'attention_mask']
						_a        =  LongformerTokenizer


						def __init__(							self			:     Optional[Any]  ,  lowerCAmelCase			:     Tuple=None  ,  lowerCAmelCase			:     Any=None  ,  lowerCAmelCase			:     List[Any]=None  ,  lowerCAmelCase			:     Optional[Any]="replace"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     int="</s>"  ,  lowerCAmelCase			:     Any="</s>"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     List[str]="<unk>"  ,  lowerCAmelCase			:     Optional[Any]="<pad>"  ,  lowerCAmelCase			:     List[str]="<mask>"  ,  lowerCAmelCase			:     Tuple=False  ,  lowerCAmelCase			:     List[str]=True  ,  **lowerCAmelCase			:     Optional[int]  ,  ):
										super().__init__(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  tokenizer_file=lowerCAmelCase  ,  errors=lowerCAmelCase  ,  bos_token=lowerCAmelCase  ,  eos_token=lowerCAmelCase  ,  sep_token=lowerCAmelCase  ,  cls_token=lowerCAmelCase  ,  unk_token=lowerCAmelCase  ,  pad_token=lowerCAmelCase  ,  mask_token=lowerCAmelCase  ,  add_prefix_space=lowerCAmelCase  ,  trim_offsets=lowerCAmelCase  ,  **lowerCAmelCase  ,  )

										lowerCAmelCase		=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
										if pre_tok_state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
														lowerCAmelCase		=			getattr(lowerCAmelCase  ,  pre_tok_state.pop("""type"""   )   )
														lowerCAmelCase		=			add_prefix_space
														lowerCAmelCase		=			pre_tok_class(**lowerCAmelCase   )

										lowerCAmelCase		=			add_prefix_space

										lowerCAmelCase		=			"""post_processor"""
										lowerCAmelCase		=			getattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										if tokenizer_component_instance:
														lowerCAmelCase		=			json.loads(tokenizer_component_instance.__getstate__()   )

														# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
														if "sep" in state:
																		lowerCAmelCase		=			tuple(state["""sep"""]   )
														if "cls" in state:
																		lowerCAmelCase		=			tuple(state["""cls"""]   )

														lowerCAmelCase		=			False

														if state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
																		lowerCAmelCase		=			add_prefix_space
																		lowerCAmelCase		=			True

														if state.get("""trim_offsets"""  ,  lowerCAmelCase   ) != trim_offsets:
																		lowerCAmelCase		=			trim_offsets
																		lowerCAmelCase		=			True

														if changes_to_apply:
																		lowerCAmelCase		=			getattr(lowerCAmelCase  ,  state.pop("""type"""   )   )
																		lowerCAmelCase		=			component_class(**lowerCAmelCase   )
																		setattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )


						@property
						def 					__lowercase	(							self			:     List[Any]   ):
										if self._mask_token is None:
														if self.verbose:
																		logger.error("""Using mask_token, but it is not set yet."""   )
														return None
										return str(self._mask_token   )


						@mask_token.setter
						def 					__lowercase	(							self			:     List[Any]  ,  lowerCAmelCase			:     Dict   ):
										lowerCAmelCase		=			AddedToken(lowerCAmelCase  ,  lstrip=lowerCAmelCase  ,  rstrip=lowerCAmelCase   ) if isinstance(lowerCAmelCase  ,  lowerCAmelCase   ) else value
										lowerCAmelCase		=			value


						def 					__lowercase	(							self			:     Tuple  ,  *lowerCAmelCase			:     Optional[int]  ,  **lowerCAmelCase			:     Optional[Any]   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     str  ,  *lowerCAmelCase			:     str  ,  **lowerCAmelCase			:     str   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     str  ,  lowerCAmelCase			:     Optional[str] = None   ):
										lowerCAmelCase		=			self._tokenizer.model.save(lowerCAmelCase  ,  name=lowerCAmelCase   )
										return tuple(lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[Any]  ,  lowerCAmelCase			:     List[str]  ,  lowerCAmelCase			:     Optional[int]=None   ):
										lowerCAmelCase		=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
														return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     List[int]  ,  lowerCAmelCase			:     Optional[List[int]] = None   ):
										lowerCAmelCase		=			[self.sep_token_id]
										lowerCAmelCase		=			[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





