




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase_		=							logging.get_logger(__name__)

lowercase_		=							{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowercase_		=							{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

lowercase_		=							{
    '''openai-gpt''': 512,
}







def 			lowerCAmelCase  (     UpperCAmelCase	)  ->Tuple:






					"""simple docstring"""






					__magic_name__ :							Optional[int]		       =							text.strip()
					if not text:
										return []
					__magic_name__ :							Any		       =							text.split()
					return tokens





class 		A__    (       snake_case__			):






	def __init__(     self    ,					lowerCamelCase=True    ,					lowerCamelCase=None    ,					lowerCamelCase=True    ,					lowerCamelCase=None    ,					lowerCamelCase=True    ,					)				->       List[Any]:




						"""simple docstring"""






						if never_split is None:
											__magic_name__ :							Any		       =							[]
						__magic_name__ :							List[Any]		       =							do_lower_case
						__magic_name__ :							Optional[int]		       =							set(_A						)
						__magic_name__ :							Tuple		       =							tokenize_chinese_chars
						__magic_name__ :							List[Any]		       =							strip_accents
						__magic_name__ :							Dict		       =							do_split_on_punc






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase=None						)				->       List[Any]:




						"""simple docstring"""






						__magic_name__ :							Union[str, Any]		       =							self.never_split.union(set(_A						)						) if never_split else self.never_split
						__magic_name__ :							Any		       =							self._clean_text(_A						)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
											__magic_name__ :							Any		       =							self._tokenize_chinese_chars(_A						)
						# prevents treating the same character with different unicode codepoints as different characters
						__magic_name__ :							List[str]		       =							unicodedata.normalize('''NFC'''    ,					_A						)
						__magic_name__ :							Any		       =							whitespace_tokenize(_A						)
						__magic_name__ :							Dict		       =							[]
						for token in orig_tokens:
											if token not in never_split:
																if self.do_lower_case:
																					__magic_name__ :							List[Any]		       =							token.lower()
																					if self.strip_accents is not False:
																										__magic_name__ :							List[str]		       =							self._run_strip_accents(_A						)
																elif self.strip_accents:
																					__magic_name__ :							Union[str, Any]		       =							self._run_strip_accents(_A						)
											split_tokens.extend(self._run_split_on_punc(_A    ,					_A						)						)

						__magic_name__ :							List[str]		       =							whitespace_tokenize(''' '''.join(_A						)						)
						return output_tokens






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Optional[Any]:




						"""simple docstring"""






						__magic_name__ :							List[Any]		       =							unicodedata.normalize('''NFD'''    ,					_A						)
						__magic_name__ :							Tuple		       =							[]
						for char in text:
											__magic_name__ :							str		       =							unicodedata.category(_A						)
											if cat == "Mn":
																continue
											output.append(_A						)
						return "".join(_A						)






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase=None						)				->       int:




						"""simple docstring"""






						if not self.do_split_on_punc or (never_split is not None and text in never_split):
											return [text]
						__magic_name__ :							Any		       =							list(_A						)
						__magic_name__ :							Union[str, Any]		       =							0
						__magic_name__ :							Union[str, Any]		       =							True
						__magic_name__ :							Dict		       =							[]
						while i < len(_A						):
											__magic_name__ :							Union[str, Any]		       =							chars[i]
											if _is_punctuation(_A						):
																output.append([char]						)
																__magic_name__ :							Union[str, Any]		       =							True
											else:
																if start_new_word:
																					output.append([]						)
																__magic_name__ :							Dict		       =							False
																output[-1].append(_A						)
											i += 1

						return ["".join(_A						) for x in output]






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Any:




						"""simple docstring"""






						__magic_name__ :							Optional[int]		       =							[]
						for char in text:
											__magic_name__ :							Union[str, Any]		       =							ord(_A						)
											if self._is_chinese_char(_A						):
																output.append(''' '''						)
																output.append(_A						)
																output.append(''' '''						)
											else:
																output.append(_A						)
						return "".join(_A						)






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Union[str, Any]:




						"""simple docstring"""






						if (
						    (cp >= 0X4e00 and cp <= 0X9fff)
						    or (cp >= 0X3400 and cp <= 0X4dbf)  #
						    or (cp >= 0X20000 and cp <= 0X2a6df)  #
						    or (cp >= 0X2a700 and cp <= 0X2b73f)  #
						    or (cp >= 0X2b740 and cp <= 0X2b81f)  #
						    or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
						    or (cp >= 0Xf900 and cp <= 0Xfaff)
						    or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
						):  #
											return True

						return False






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Optional[Any]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							[]
						for char in text:
											__magic_name__ :							Tuple		       =							ord(_A						)
											if cp == 0 or cp == 0Xfffd or _is_control(_A						):
																continue
											if _is_whitespace(_A						):
																output.append(''' '''						)
											else:
																output.append(_A						)
						return "".join(_A						)







def 			lowerCAmelCase  (     UpperCAmelCase	)  ->int:






					"""simple docstring"""






					__magic_name__ :							List[str]		       =							set()
					__magic_name__ :							Optional[int]		       =							word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										__magic_name__ :							Union[str, Any]		       =							char
					return pairs







def 			lowerCAmelCase  (     UpperCAmelCase	)  ->List[Any]:






					"""simple docstring"""






					__magic_name__ :							int		       =							text.replace('''—''',     '''-'''	)
					__magic_name__ :							Union[str, Any]		       =							text.replace('''–''',     '''-'''	)
					__magic_name__ :							str		       =							text.replace('''―''',     '''-'''	)
					__magic_name__ :							int		       =							text.replace('''…''',     '''...'''	)
					__magic_name__ :							Optional[int]		       =							text.replace('''´''',     '''\''''	)
					__magic_name__ :							Dict		       =							re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)''',     r''' \1 ''',     UpperCAmelCase	)
					__magic_name__ :							Optional[int]		       =							re.sub(r'''\s*\n\s*''',     ''' \n ''',     UpperCAmelCase	)
					__magic_name__ :							List[Any]		       =							re.sub(r'''[^\S\n]+''',     ''' ''',     UpperCAmelCase	)
					return text.strip()





class 		A__    (       snake_case__			):
	lowerCamelCase__		:			int       =VOCAB_FILES_NAMES
	lowerCamelCase__		:			Union[str, Any]       =PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__		:			Any       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__		:			Dict       =['''input_ids''', '''attention_mask''']






	def __init__(     self    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase="<unk>"    ,					**lowerCamelCase						)				->       str:




						"""simple docstring"""






						super().__init__(unk_token=_A    ,					**_A						)

						try:
											import ftfy
											from spacy.lang.en import English

											__magic_name__ :							List[str]		       =							English()
											__magic_name__ :							Tuple		       =							_nlp.tokenizer
											__magic_name__ :							Dict		       =							ftfy.fix_text
						except ImportError:
											logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''						)
											__magic_name__ :							List[str]		       =							BasicTokenizer(do_lower_case=_A						)
											__magic_name__ :							str		       =							None

						with open(_A    ,					encoding='''utf-8'''						) as vocab_handle:
											__magic_name__ :							Union[str, Any]		       =							json.load(_A						)
						__magic_name__ :							List[Any]		       =							{v: k for k, v in self.encoder.items()}
						with open(_A    ,					encoding='''utf-8'''						) as merges_handle:
											__magic_name__ :							List[str]		       =							merges_handle.read().split('''\n'''						)[1:-1]
						__magic_name__ :							List[Any]		       =							[tuple(merge.split()						) for merge in merges]
						__magic_name__ :							Dict		       =							dict(zip(_A    ,					range(len(_A						)						)						)						)
						__magic_name__ :							Any		       =							{}






	@property
	def 			lowercase				(     self						)				->       Dict:




						"""simple docstring"""






						return True






	@property
	def 			lowercase				(     self						)				->       Any:




						"""simple docstring"""






						return len(self.encoder						)






	def 			lowercase				(     self						)				->       List[str]:




						"""simple docstring"""






						return dict(self.encoder    ,					**self.added_tokens_encoder						)






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Any:




						"""simple docstring"""






						__magic_name__ :							Optional[int]		       =							tuple(token[:-1]						) + (token[-1] + '</w>',)
						if token in self.cache:
											return self.cache[token]
						__magic_name__ :							Any		       =							get_pairs(_A						)

						if not pairs:
											return token + "</w>"

						while True:
											__magic_name__ :							Tuple		       =							min(_A    ,					key=lambda lowerCamelCase						: self.bpe_ranks.get(_A    ,					float('''inf'''						)						)						)
											if bigram not in self.bpe_ranks:
																break
											__magic_name__ :							Optional[Any]		       =							bigram
											__magic_name__ :							Dict		       =							[]
											__magic_name__ :							List[str]		       =							0
											while i < len(_A						):
																try:
																					__magic_name__ :							Union[str, Any]		       =							word.index(_A    ,					_A						)
																except ValueError:
																					new_word.extend(word[i:]						)
																					break
																else:
																					new_word.extend(word[i:j]						)
																					__magic_name__ :							List[str]		       =							j

																if word[i] == first and i < len(_A						) - 1 and word[i + 1] == second:
																					new_word.append(first + second						)
																					i += 2
																else:
																					new_word.append(word[i]						)
																					i += 1
											__magic_name__ :							Union[str, Any]		       =							tuple(_A						)
											__magic_name__ :							Union[str, Any]		       =							new_word
											if len(_A						) == 1:
																break
											else:
																__magic_name__ :							List[Any]		       =							get_pairs(_A						)
						__magic_name__ :							Optional[int]		       =							' '.join(_A						)
						if word == "\n  </w>":
											__magic_name__ :							Tuple		       =							'\n</w>'
						__magic_name__ :							List[Any]		       =							word
						return word






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       List[Any]:




						"""simple docstring"""






						__magic_name__ :							Union[str, Any]		       =							[]
						if self.fix_text is None:
											# Using BERT's BasicTokenizer
											__magic_name__ :							Optional[Any]		       =							self.nlp.tokenize(_A						)
											for token in text:
																split_tokens.extend(list(self.bpe(_A						).split(''' '''						)						)						)
						else:
											# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
											__magic_name__ :							Optional[int]		       =							self.nlp(text_standardize(self.fix_text(_A						)						)						)
											for token in text:
																split_tokens.extend(list(self.bpe(token.text.lower()						).split(''' '''						)						)						)
						return split_tokens






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Union[str, Any]:




						"""simple docstring"""






						return self.encoder.get(_A    ,					self.encoder.get(self.unk_token						)						)






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       List[str]:




						"""simple docstring"""






						return self.decoder.get(_A    ,					self.unk_token						)






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       str:




						"""simple docstring"""






						__magic_name__ :							Tuple		       =							''.join(_A						).replace('''</w>'''    ,					''' '''						).strip()
						return out_string






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase = None						)				->       List[str]:




						"""simple docstring"""






						if not os.path.isdir(_A						):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
						__magic_name__ :							Optional[int]		       =							os.path.join(
						    _A    ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
						__magic_name__ :							Any		       =							os.path.join(
						    _A    ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

						with open(_A    ,					'''w'''    ,					encoding='''utf-8'''						) as f:
											f.write(json.dumps(self.encoder    ,					indent=2    ,					sort_keys=_A    ,					ensure_ascii=_A						) + '''\n'''						)

						__magic_name__ :							Optional[Any]		       =							0
						with open(_A    ,					'''w'''    ,					encoding='''utf-8'''						) as writer:
											writer.write('''#version: 0.2\n'''						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,					key=lambda lowerCamelCase						: kv[1]						):
																if index != token_index:
																					logger.warning(
																					    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																					    ''' Please check that the tokenizer is not corrupted!'''						)
																					__magic_name__ :							Optional[int]		       =							token_index
																writer.write(''' '''.join(_A						) + '''\n'''						)
																index += 1

						return vocab_file, merge_file









from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowercase_		=							logging.get_logger(__name__)

lowercase_		=							{
    '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json''',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class 		A__    (       __SCREAMING_SNAKE_CASE			):
	lowerCamelCase__		:			Optional[int]       ="m2m_100"
	lowerCamelCase__		:			List[str]       =["past_key_values"]
	lowerCamelCase__		:			Union[str, Any]       ={"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}






	def __init__(     self    ,					lowerCamelCase=128112    ,					lowerCamelCase=1024    ,					lowerCamelCase=12    ,					lowerCamelCase=4096    ,					lowerCamelCase=16    ,					lowerCamelCase=12    ,					lowerCamelCase=4096    ,					lowerCamelCase=16    ,					lowerCamelCase=0.0_5    ,					lowerCamelCase=0.0_5    ,					lowerCamelCase=True    ,					lowerCamelCase=True    ,					lowerCamelCase="relu"    ,					lowerCamelCase=1024    ,					lowerCamelCase=0.1    ,					lowerCamelCase=0.1    ,					lowerCamelCase=0.0    ,					lowerCamelCase=0.0_2    ,					lowerCamelCase=2    ,					lowerCamelCase=True    ,					lowerCamelCase=1    ,					lowerCamelCase=0    ,					lowerCamelCase=2    ,					**lowerCamelCase    ,					)				->       List[str]:




						"""simple docstring"""






						__magic_name__ :							List[Any]		       =							vocab_size
						__magic_name__ :							str		       =							max_position_embeddings
						__magic_name__ :							Union[str, Any]		       =							d_model
						__magic_name__ :							Dict		       =							encoder_ffn_dim
						__magic_name__ :							List[Any]		       =							encoder_layers
						__magic_name__ :							Optional[int]		       =							encoder_attention_heads
						__magic_name__ :							Dict		       =							decoder_ffn_dim
						__magic_name__ :							List[str]		       =							decoder_layers
						__magic_name__ :							Optional[Any]		       =							decoder_attention_heads
						__magic_name__ :							List[Any]		       =							dropout
						__magic_name__ :							List[str]		       =							attention_dropout
						__magic_name__ :							Optional[int]		       =							activation_dropout
						__magic_name__ :							Dict		       =							activation_function
						__magic_name__ :							Tuple		       =							init_std
						__magic_name__ :							Optional[int]		       =							encoder_layerdrop
						__magic_name__ :							int		       =							decoder_layerdrop
						__magic_name__ :							Union[str, Any]		       =							use_cache
						__magic_name__ :							Optional[int]		       =							encoder_layers
						__magic_name__ :							Dict		       =							scale_embedding  # scale factor will be sqrt(d_model) if True

						super().__init__(
						    pad_token_id=lowerCamelCase    ,					bos_token_id=lowerCamelCase    ,					eos_token_id=lowerCamelCase    ,					is_encoder_decoder=lowerCamelCase    ,					decoder_start_token_id=lowerCamelCase    ,					**lowerCamelCase    ,					)





class 		A__    (       __SCREAMING_SNAKE_CASE			):






	@property
	def 			lowercase				(     self						)				->       Mapping[str, Mapping[int, str]]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							OrderedDict(
						    [
						        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
						        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
						    ]						)

						if self.use_past:
											__magic_name__ :							Optional[Any]		       =							{0: '''batch'''}
											__magic_name__ :							Any		       =							{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
						else:
											__magic_name__ :							Union[str, Any]		       =							{0: '''batch''', 1: '''decoder_sequence'''}
											__magic_name__ :							Optional[Any]		       =							{0: '''batch''', 1: '''decoder_sequence'''}

						if self.use_past:
											self.fill_with_past_key_values_(lowerCamelCase    ,					direction='''inputs'''						)
						return common_inputs






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase = -1    ,					lowerCamelCase = -1    ,					lowerCamelCase = False    ,					lowerCamelCase = None    ,					)				->       Mapping[str, Any]:




						"""simple docstring"""






						__magic_name__ :							int		       =							compute_effective_axis_dimension(
						    lowerCamelCase    ,					fixed_dimension=OnnxConfig.default_fixed_batch    ,					num_token_to_add=0						)

						# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
						__magic_name__ :							str		       =							tokenizer.num_special_tokens_to_add(lowerCamelCase						)
						__magic_name__ :							Optional[Any]		       =							compute_effective_axis_dimension(
						    lowerCamelCase    ,					fixed_dimension=OnnxConfig.default_fixed_sequence    ,					num_token_to_add=lowerCamelCase						)

						# Generate dummy inputs according to compute batch and sequence
						__magic_name__ :							Optional[Any]		       =							[''' '''.join([tokenizer.unk_token]						) * seq_length] * batch_size
						__magic_name__ :							Any		       =							dict(tokenizer(lowerCamelCase    ,					return_tensors=lowerCamelCase						)						)
						return common_inputs






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase = -1    ,					lowerCamelCase = -1    ,					lowerCamelCase = False    ,					lowerCamelCase = None    ,					)				->       Mapping[str, Any]:




						"""simple docstring"""






						__magic_name__ :							Any		       =							self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase						)

						# Generate decoder inputs
						__magic_name__ :							int		       =							seq_length if not self.use_past else 1
						__magic_name__ :							Optional[Any]		       =							self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase						)
						__magic_name__ :							List[str]		       =							{F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
						__magic_name__ :							Optional[Any]		       =							dict(**lowerCamelCase    ,					**lowerCamelCase						)

						if self.use_past:
											if not is_torch_available():
																raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''						)
											else:
																import torch
											__magic_name__       ,			__magic_name__ :							List[Any]		       =							common_inputs['''input_ids'''].shape
											__magic_name__ :							List[str]		       =							common_inputs['''decoder_input_ids'''].shape[1]
											__magic_name__       ,			__magic_name__ :							Any		       =							self.num_attention_heads
											__magic_name__ :							Optional[int]		       =							(
											    batch,
											    num_encoder_attention_heads,
											    encoder_seq_length,
											    self._config.hidden_size // num_encoder_attention_heads,
											)
											__magic_name__ :							Tuple		       =							decoder_seq_length + 3
											__magic_name__ :							str		       =							(
											    batch,
											    num_decoder_attention_heads,
											    decoder_past_length,
											    self._config.hidden_size // num_decoder_attention_heads,
											)

											__magic_name__ :							int		       =							torch.cat(
											    [common_inputs['''decoder_attention_mask'''], torch.ones(lowerCamelCase    ,					lowerCamelCase						)]    ,					dim=1						)

											__magic_name__ :							List[str]		       =							[]
											# If the number of encoder and decoder layers are present in the model configuration, both are considered
											__magic_name__       ,			__magic_name__ :							int		       =							self.num_layers
											__magic_name__ :							List[Any]		       =							min(lowerCamelCase    ,					lowerCamelCase						)
											__magic_name__ :							int		       =							max(lowerCamelCase    ,					lowerCamelCase						) - min_num_layers
											__magic_name__ :							Optional[int]		       =							'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

											for _ in range(lowerCamelCase						):
																common_inputs["past_key_values"].append(
																    (
																        torch.zeros(lowerCamelCase						),
																        torch.zeros(lowerCamelCase						),
																        torch.zeros(lowerCamelCase						),
																        torch.zeros(lowerCamelCase						),
																    )						)
											# TODO: test this.
											__magic_name__ :							Tuple		       =							encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
											for _ in range(lowerCamelCase    ,					lowerCamelCase						):
																common_inputs["past_key_values"].append((torch.zeros(lowerCamelCase						), torch.zeros(lowerCamelCase						))						)
						return common_inputs





	lowerCamelCase__		:			List[str]       =_generate_dummy_inputs_for_default_and_seqaseq_lm



