
import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__							:							Union[str, Any]													=     logging.get_logger(__name__)

lowercase__							:							List[Any]													=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowercase__							:							Optional[int]													=     {
    "vocab_file": {
        "microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json",
    },
    "merges_file": {"microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"},
}

lowercase__							:							Dict													=     {
    "microsoft/biogpt": 1_0_2_4,
}




def 		A_	(							snake_case							:				Dict	) ->      List[str]:



						'''simple docstring'''

						__UpperCamelCase       					=     set()
						__UpperCamelCase       					=     word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)	)
												__UpperCamelCase       					=     char
						return pairs






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							_snake_case       						=					VOCAB_FILES_NAMES
							_snake_case       						=					PRETRAINED_VOCAB_FILES_MAP
							_snake_case       						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_snake_case       						=					["input_ids", "attention_mask"]



							def __init__(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_="<unk>"       ,		SCREAMING_SNAKE_CASE_="<s>"       ,		SCREAMING_SNAKE_CASE_="</s>"       ,		SCREAMING_SNAKE_CASE_="</s>"       ,		SCREAMING_SNAKE_CASE_="<pad>"       ,		**SCREAMING_SNAKE_CASE_       ,		)->				Dict:





													'''simple docstring'''






													super().__init__(
													    bos_token=lowercase_       ,		eos_token=lowercase_       ,		sep_token=lowercase_       ,		unk_token=lowercase_       ,		pad_token=lowercase_       ,		**lowercase_       ,		)

													try:
																			import sacremoses
													except ImportError:
																			raise ImportError(
																			    '''You need to install sacremoses to use BioGptTokenizer. '''
																			    '''See https://pypi.org/project/sacremoses/ for installation.'''     )

													__UpperCamelCase       					=     '''en'''
													__UpperCamelCase       					=     sacremoses
													# cache of sm.MosesTokenizer instance
													__UpperCamelCase       					=     {}
													__UpperCamelCase       					=     {}
													with open(lowercase_       ,		encoding='''utf-8'''     ) as vocab_handle:
																			__UpperCamelCase       					=     json.load(lowercase_     )
													__UpperCamelCase       					=     {v: k for k, v in self.encoder.items()}
													with open(lowercase_       ,		encoding='''utf-8'''     ) as merges_handle:
																			__UpperCamelCase       					=     merges_handle.read().split('''\n'''     )[:-1]
													__UpperCamelCase       					=     [tuple(merge.split()[:2]     ) for merge in merges]
													__UpperCamelCase       					=     dict(zip(lowercase_       ,		range(len(lowercase_     )     )     )     )
													__UpperCamelCase       					=     {}



							@property
							def       A__						(					self     )->				str:





													'''simple docstring'''






													return len(self.encoder     )



							def       A__						(					self     )->				Optional[Any]:





													'''simple docstring'''






													return dict(self.encoder       ,		**self.added_tokens_encoder     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_     )->				Dict:





													'''simple docstring'''






													if lang not in self.cache_moses_tokenizer:
																			__UpperCamelCase       					=     self.sm.MosesTokenizer(lang=lowercase_     )
																			__UpperCamelCase       					=     moses_tokenizer
													return self.cache_moses_tokenizer[lang].tokenize(
													    lowercase_       ,		aggressive_dash_splits=lowercase_       ,		return_str=lowercase_       ,		escape=lowercase_     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_     )->				Optional[Any]:





													'''simple docstring'''






													if lang not in self.cache_moses_detokenizer:
																			__UpperCamelCase       					=     self.sm.MosesDetokenizer(lang=lowercase_     )
																			__UpperCamelCase       					=     moses_detokenizer
													return self.cache_moses_detokenizer[lang].detokenize(lowercase_     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				List[Any]:





													'''simple docstring'''






													__UpperCamelCase       					=     tuple(token[:-1]     ) + (token[-1] + '''</w>''',)
													if token in self.cache:
																			return self.cache[token]
													__UpperCamelCase       					=     get_pairs(lowercase_     )

													if not pairs:
																			return token + "</w>"

													while True:
																			__UpperCamelCase       					=     min(lowercase_       ,		key=lambda SCREAMING_SNAKE_CASE_     : self.bpe_ranks.get(lowercase_       ,		float('''inf'''     )     )     )
																			if bigram not in self.bpe_ranks:
																									break
																			__UpperCamelCase					,							__UpperCamelCase       					=     bigram
																			__UpperCamelCase       					=     []
																			__UpperCamelCase       					=     0
																			while i < len(lowercase_     ):
																									try:
																															__UpperCamelCase       					=     word.index(lowercase_       ,		lowercase_     )
																									except ValueError:
																															new_word.extend(word[i:]     )
																															break
																									else:
																															new_word.extend(word[i:j]     )
																															__UpperCamelCase       					=     j

																									if word[i] == first and i < len(lowercase_     ) - 1 and word[i + 1] == second:
																															new_word.append(first + second     )
																															i += 2
																									else:
																															new_word.append(word[i]     )
																															i += 1
																			__UpperCamelCase       					=     tuple(lowercase_     )
																			__UpperCamelCase       					=     new_word
																			if len(lowercase_     ) == 1:
																									break
																			else:
																									__UpperCamelCase       					=     get_pairs(lowercase_     )
													__UpperCamelCase       					=     ''' '''.join(lowercase_     )
													if word == "\n  </w>":
																			__UpperCamelCase       					=     '''\n</w>'''
													__UpperCamelCase       					=     word
													return word



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_=False     )->				Dict:





													'''simple docstring'''






													if bypass_tokenizer:
																			__UpperCamelCase       					=     text.split()
													else:
																			__UpperCamelCase       					=     self.moses_tokenize(lowercase_       ,		self.lang     )

													__UpperCamelCase       					=     []
													for token in text:
																			if token:
																									split_tokens.extend(list(self.bpe(lowercase_     ).split(''' '''     )     )     )

													return split_tokens



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				Optional[Any]:





													'''simple docstring'''






													return self.encoder.get(lowercase_       ,		self.encoder.get(self.unk_token     )     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				Any:





													'''simple docstring'''






													return self.decoder.get(lowercase_       ,		self.unk_token     )



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_     )->				Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase       					=     [t.replace(''' '''       ,		''''''     ).replace('''</w>'''       ,		''' '''     ) for t in tokens]
													__UpperCamelCase       					=     ''''''.join(lowercase_     ).split()
													# detokenize
													__UpperCamelCase       					=     self.moses_detokenize(lowercase_       ,		self.lang     )
													return text



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				List[int]:





													'''simple docstring'''






													if token_ids_a is None:
																			return [self.sep_token_id] + token_ids_a
													__UpperCamelCase       					=     [self.sep_token_id]
													return sep + token_ids_a + sep + token_ids_a



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None       ,		SCREAMING_SNAKE_CASE_ = False     )->				List[int]:





													'''simple docstring'''






													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=lowercase_       ,		token_ids_a=lowercase_       ,		already_has_special_tokens=lowercase_     )
													# no bos used in fairseq
													if token_ids_a is not None:
																			return [1] + ([0] * len(lowercase_     )) + [1] + ([0] * len(lowercase_     ))
													return [1] + ([0] * len(lowercase_     ))



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				List[int]:





													'''simple docstring'''






													__UpperCamelCase       					=     [self.sep_token_id]

													# no bos used in fairseq
													if token_ids_a is None:
																			return len(token_ids_a + sep     ) * [0]
													return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



							def       A__						(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_ = None     )->				Tuple[str]:





													'''simple docstring'''






													if not os.path.isdir(lowercase_     ):
																			logger.error(F"Vocabulary path ({save_directory}) should be a directory"     )
																			return
													__UpperCamelCase       					=     os.path.join(
													    lowercase_       ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
													__UpperCamelCase       					=     os.path.join(
													    lowercase_       ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

													with open(lowercase_       ,		'''w'''       ,		encoding='''utf-8'''     ) as f:
																			f.write(json.dumps(self.encoder       ,		indent=2       ,		sort_keys=lowercase_       ,		ensure_ascii=lowercase_     ) + '''\n'''     )

													__UpperCamelCase       					=     0
													with open(lowercase_       ,		'''w'''       ,		encoding='''utf-8'''     ) as writer:
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,		key=lambda SCREAMING_SNAKE_CASE_     : kv[1]     ):
																									if index != token_index:
																															logger.warning(
																															    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																															    ''' Please check that the tokenizer is not corrupted!'''     )
																															__UpperCamelCase       					=     token_index
																									writer.write(''' '''.join(lowercase_     ) + '''\n'''     )
																									index += 1

													return vocab_file, merge_file



							def __getstate__(					self     )->				Dict:





													'''simple docstring'''






													__UpperCamelCase       					=     self.__dict__.copy()
													__UpperCamelCase       					=     None
													return state



							def __setstate__(					self       ,		SCREAMING_SNAKE_CASE_     )->				int:





													'''simple docstring'''






													__UpperCamelCase       					=     d

													try:
																			import sacremoses
													except ImportError:
																			raise ImportError(
																			    '''You need to install sacremoses to use XLMTokenizer. '''
																			    '''See https://pypi.org/project/sacremoses/ for installation.'''     )

													__UpperCamelCase       					=     sacremoses






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available




def 		A_	(							snake_case							:				Union[str, Any]	, snake_case							:				str	, snake_case							:				Dict	, snake_case							:				Optional[Any]=None	) ->      Optional[Any]:



						'''simple docstring'''

						__UpperCamelCase       					=     None
						# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
						if str(weight.dtype	) == "torch.bfloat16":
												# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
												__UpperCamelCase       					=     weight.view(torch.intaa	)
												__UpperCamelCase       					=     '''bfloat16'''
						__UpperCamelCase       					=     weight.cpu().numpy()
						__UpperCamelCase       					=     os.path.join(snake_case	, f"{weight_name}.dat"	)
						if index is not None:
												if dtype is None:
																		__UpperCamelCase       					=     str(array.dtype	)
												__UpperCamelCase       					=     {'''dtype''': dtype, '''shape''': list(array.shape	)}
						if array.ndim == 0:
												__UpperCamelCase       					=     array[None]
						__UpperCamelCase       					=     np.memmap(snake_case	, dtype=array.dtype	, mode='''w+'''	, shape=array.shape	)
						__UpperCamelCase       					=     array[:]
						file_array.flush()
						return index




def 		A_	(							snake_case							:				Tuple	, snake_case							:				Union[str, Any]	) ->      Dict:



						'''simple docstring'''

						__UpperCamelCase       					=     tuple(weight_info['''shape''']	)
						if shape == ():
												# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
												__UpperCamelCase       					=     (1,)

						__UpperCamelCase       					=     weight_info['''dtype''']
						if dtype == "bfloat16":
												# NumPy does not support bfloat16 so this was saved as a int16
												__UpperCamelCase       					=     '''int16'''

						__UpperCamelCase       					=     np.memmap(snake_case	, dtype=snake_case	, shape=snake_case	, mode='''r'''	)

						if len(weight_info['''shape''']	) == 0:
												__UpperCamelCase       					=     weight[0]
						__UpperCamelCase       					=     torch.tensor(snake_case	)
						if weight_info["dtype"] == "bfloat16":
												__UpperCamelCase       					=     weight.view(torch.bfloataa	)

						return weight




def 		A_	(							snake_case							:				Dict	, snake_case							:				int	) ->      str:



						'''simple docstring'''

						if index is None or len(snake_case	) == 0:
												# Nothing to save
												return

						__UpperCamelCase       					=     os.path.join(snake_case	, '''index.json'''	)
						if os.path.isfile(snake_case	):
												with open(snake_case	, '''r'''	, encoding='''utf-8'''	) as f:
																		__UpperCamelCase       					=     json.load(snake_case	)
						else:
												__UpperCamelCase       					=     {}
						current_index.update(snake_case	)

						with open(snake_case	, '''w'''	, encoding='''utf-8'''	) as f:
												json.dump(snake_case	, snake_case	, indent=2	)




def 		A_	(							snake_case							:				Union[str, os.PathLike]	, snake_case							:				Dict[str, torch.Tensor]	) ->      Dict:



						'''simple docstring'''

						os.makedirs(snake_case	, exist_ok=snake_case	)
						__UpperCamelCase       					=     {}
						for name, parameter in state_dict.items():
												__UpperCamelCase       					=     offload_weight(snake_case	, snake_case	, snake_case	, index=snake_case	)

						# Update index
						save_offload_index(snake_case	, snake_case	)






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							def __init__(					self       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_     )->				str:





													'''simple docstring'''






													__UpperCamelCase       					=     dataset
													__UpperCamelCase       					=     prefix



							def __getitem__(					self       ,		SCREAMING_SNAKE_CASE_     )->				Union[str, Any]:





													'''simple docstring'''






													return self.dataset[F"{self.prefix}{key}"]



							def __iter__(					self     )->				Union[str, Any]:





													'''simple docstring'''






													return iter([key for key in self.dataset if key.startswith(self.prefix     )]     )



							def __len__(					self     )->				str:





													'''simple docstring'''






													return len(self.dataset     )






class     SCREAMING_SNAKE_CASE__      (							SCREAMING_SNAKE_CASE_      ):

							"""simple docstring"""






							def __init__(					self       ,		SCREAMING_SNAKE_CASE_ = None       ,		SCREAMING_SNAKE_CASE_ = None       ,		SCREAMING_SNAKE_CASE_ = None       ,		SCREAMING_SNAKE_CASE_=None       ,		)->				Tuple:





													'''simple docstring'''






													if state_dict is None and save_folder is None:
																			raise ValueError('''Need either a `state_dict` or a `save_folder` containing offloaded weights.'''     )

													__UpperCamelCase       					=     {} if state_dict is None else state_dict
													__UpperCamelCase       					=     save_folder
													if index is None and save_folder is not None:
																			with open(os.path.join(SCREAMING_SNAKE_CASE_       ,		'''index.json'''     )     ) as f:
																									__UpperCamelCase       					=     json.load(SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     {} if index is None else index
													__UpperCamelCase       					=     list(self.state_dict.keys()     )
													self.all_keys.extend([key for key in self.index if key not in self.all_keys]     )
													__UpperCamelCase       					=     device



							def __getitem__(					self       ,		SCREAMING_SNAKE_CASE_     )->				Tuple:





													'''simple docstring'''






													if key in self.state_dict:
																			return self.state_dict[key]
													__UpperCamelCase       					=     self.index[key]
													if weight_info.get('''safetensors_file'''     ) is not None:
																			if not is_safetensors_available():
																									raise ImportError('''These offloaded weights require the use of safetensors: `pip install safetensors`.'''     )

																			from safetensors import safe_open

																			__UpperCamelCase       					=     '''cpu''' if self.device is None else self.device
																			with safe_open(weight_info['''safetensors_file''']       ,		framework='''pt'''       ,		device=SCREAMING_SNAKE_CASE_     ) as f:
																									__UpperCamelCase       					=     f.get_tensor(weight_info.get('''weight_name'''       ,		SCREAMING_SNAKE_CASE_     )     )

																			if "dtype" in weight_info:
																									return tensor.to(getattr(SCREAMING_SNAKE_CASE_       ,		weight_info['''dtype''']     )     )
																			else:
																									return tensor

													__UpperCamelCase       					=     os.path.join(self.save_folder       ,		F"{key}.dat"     )
													return load_offloaded_weight(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_     )



							def __iter__(					self     )->				Dict:





													'''simple docstring'''






													return iter(self.all_keys     )



							def __len__(					self     )->				Dict:





													'''simple docstring'''






													return len(self.all_keys     )





def 		A_	(							snake_case							:				Dict[str, torch.Tensor]	, snake_case							:				List[str]	) ->      Tuple:



						'''simple docstring'''

						__UpperCamelCase       					=     {}
						for module_name in submodule_names:
												# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
												# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
												result.update(
												    {
												        key: param
												        for key, param in state_dict.items()
												        if key == module_name or key.startswith(module_name + '''.'''	)
												    }	)
						return result




