




'''simple docstring'''







import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available






def 		_lowercase		(     lowercase_						,							lowercase_						,							lowercase_						,							lowercase_=None  ):



		'''simple docstring'''







		_snake_case     	=	None
		# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
		if str(weight.dtype  ) == "torch.bfloat16":
				# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
				_snake_case     	=	weight.view(torch.intaa  )
				_snake_case     	=	"bfloat16"
		_snake_case     	=	weight.cpu().numpy()
		_snake_case     	=	os.path.join(lowercase_						,							f'''{weight_name}.dat'''  )
		if index is not None:
				if dtype is None:
						_snake_case     	=	str(array.dtype  )
				_snake_case     	=	{"dtype": dtype, "shape": list(array.shape  )}
		if array.ndim == 0:
				_snake_case     	=	array[None]
		_snake_case     	=	np.memmap(lowercase_						,							dtype=array.dtype						,							mode="w+"						,							shape=array.shape  )
		_snake_case     	=	array[:]
		file_array.flush()
		return index






def 		_lowercase		(     lowercase_						,							lowercase_  ):



		'''simple docstring'''







		_snake_case     	=	tuple(weight_info["shape"]  )
		if shape == ():
				# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
				_snake_case     	=	(1,)

		_snake_case     	=	weight_info["dtype"]
		if dtype == "bfloat16":
				# NumPy does not support bfloat16 so this was saved as a int16
				_snake_case     	=	"int16"

		_snake_case     	=	np.memmap(lowercase_						,							dtype=lowercase_						,							shape=lowercase_						,							mode="r"  )

		if len(weight_info["shape"]  ) == 0:
				_snake_case     	=	weight[0]
		_snake_case     	=	torch.tensor(lowercase_  )
		if weight_info["dtype"] == "bfloat16":
				_snake_case     	=	weight.view(torch.bfloataa  )

		return weight






def 		_lowercase		(     lowercase_						,							lowercase_  ):



		'''simple docstring'''







		if index is None or len(lowercase_  ) == 0:
				# Nothing to save
				return

		_snake_case     	=	os.path.join(lowercase_						,							"index.json"  )
		if os.path.isfile(lowercase_  ):
				with open(lowercase_						,							"r"						,							encoding="utf-8"  ) as f:
						_snake_case     	=	json.load(lowercase_  )
		else:
				_snake_case     	=	{}
		current_index.update(lowercase_  )

		with open(lowercase_						,							"w"						,							encoding="utf-8"  ) as f:
				json.dump(lowercase_						,							lowercase_						,							indent=2  )






def 		_lowercase		(     lowercase_						,							lowercase_  ):



		'''simple docstring'''







		os.makedirs(lowercase_						,							exist_ok=lowercase_  )
		_snake_case     	=	{}
		for name, parameter in state_dict.items():
				_snake_case     	=	offload_weight(lowercase_						,							lowercase_						,							lowercase_						,							index=lowercase_  )

		# Update index
		save_offload_index(lowercase_						,							lowercase_  )





class 					lowerCamelCase_							(	UpperCAmelCase_							):




					'''simple docstring'''






					def __init__(   self,  __UpperCamelCase,  __UpperCamelCase			):
							_snake_case     	=	dataset
							_snake_case     	=	prefix






					def __getitem__(   self,  __UpperCamelCase			):
							return self.dataset[f'''{self.prefix}{key}''']






					def __iter__(   self			):
							return iter([key for key in self.dataset if key.startswith(self.prefix			)]			)






					def __len__(   self			):
							return len(self.dataset			)





class 					lowerCamelCase_							(	UpperCAmelCase_							):




					'''simple docstring'''






					def __init__(   self,  __UpperCamelCase = None,  __UpperCamelCase = None,  __UpperCamelCase = None,  __UpperCamelCase=None,  ):
							if state_dict is None and save_folder is None:
									raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."			)

							_snake_case     	=	{} if state_dict is None else state_dict
							_snake_case     	=	save_folder
							if index is None and save_folder is not None:
									with open(os.path.join(__UpperCamelCase,  "index.json"			)			) as f:
											_snake_case     	=	json.load(__UpperCamelCase			)
							_snake_case     	=	{} if index is None else index
							_snake_case     	=	list(self.state_dict.keys()			)
							self.all_keys.extend([key for key in self.index if key not in self.all_keys]			)
							_snake_case     	=	device






					def __getitem__(   self,  __UpperCamelCase			):
							# State dict gets priority
							if key in self.state_dict:
									return self.state_dict[key]
							_snake_case     	=	self.index[key]
							if weight_info.get("safetensors_file"			) is not None:
									if not is_safetensors_available():
											raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."			)

									from safetensors import safe_open

									_snake_case     	=	"cpu" if self.device is None else self.device
									with safe_open(weight_info["safetensors_file"],  framework="pt",  device=__UpperCamelCase			) as f:
											_snake_case     	=	f.get_tensor(weight_info.get("weight_name",  __UpperCamelCase			)			)

									if "dtype" in weight_info:
											return tensor.to(getattr(__UpperCamelCase,  weight_info["dtype"]			)			)
									else:
											return tensor

							_snake_case     	=	os.path.join(self.save_folder,  f'''{key}.dat'''			)
							return load_offloaded_weight(__UpperCamelCase,  __UpperCamelCase			)






					def __iter__(   self			):
							return iter(self.all_keys			)






					def __len__(   self			):
							return len(self.all_keys			)






def 		_lowercase		(     lowercase_						,							lowercase_  ):



		'''simple docstring'''







		_snake_case     	=	{}
		for module_name in submodule_names:
				# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
				# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
				result.update(
				    {
				        key: param
				        for key, param in state_dict.items()
				        if key == module_name or key.startswith(module_name + "."  )
				    }  )
		return result











'''simple docstring'''







import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase





@require_tokenizers
class 					lowerCamelCase_							(	unittest.TestCase							):




					'''simple docstring'''






					lowercase__   =			EsmTokenizer






					def     __SCREAMING_SNAKE_CASE     (   self			):
							super().setUp()
							_snake_case     	=	tempfile.mkdtemp()
							# fmt: off
							_snake_case     	=	["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
							# fmt: on
							_snake_case     	=	os.path.join(self.tmpdirname,  VOCAB_FILES_NAMES["vocab_file"]			)
							with open(self.vocab_file,  "w",  encoding="utf-8"			) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)






					def     __SCREAMING_SNAKE_CASE     (   self,  **__UpperCamelCase			):
							return [self.get_tokenizer(**__UpperCamelCase			)]






					def     __SCREAMING_SNAKE_CASE     (   self,  **__UpperCamelCase			):
							return self.tokenizer_class.from_pretrained(self.tmpdirname,  **__UpperCamelCase			)






					def     __SCREAMING_SNAKE_CASE     (   self			):
							_snake_case     	=	self.tokenizer_class(self.vocab_file			)

							_snake_case     	=	tokenizer.tokenize("LAGVS"			)
							self.assertListEqual(__UpperCamelCase,  ["L", "A", "G", "V", "S"]			)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase			),  [4, 5, 6, 7, 8]			)






					def     __SCREAMING_SNAKE_CASE     (   self			):
							_snake_case     	=	self.tokenizer_class(self.vocab_file			)

							_snake_case     	=	"LAGVS"
							self.assertListEqual(tokenizer.encode(__UpperCamelCase			),  [0, 4, 5, 6, 7, 8, 2]			)






					def     __SCREAMING_SNAKE_CASE     (   self			):
							_snake_case     	=	self.tokenizer_class(self.vocab_file			)

							_snake_case     	=	["LAGVS", "WCB"]
							_snake_case     	=	tokenizer(__UpperCamelCase,  padding=__UpperCamelCase			)["input_ids"]

							self.assertListEqual(__UpperCamelCase,  [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]			)






					def     __SCREAMING_SNAKE_CASE     (   self			):
							_snake_case     	=	self.tokenizer_class(self.vocab_file			)

							_snake_case     	=	["LAGVS", "WCB"]
							_snake_case     	=	tokenizer(__UpperCamelCase,  padding=__UpperCamelCase			)["input_ids"]

							self.assertListEqual(__UpperCamelCase,  [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]			)






					def     __SCREAMING_SNAKE_CASE     (   self			):
							_snake_case     	=	self.get_tokenizers(fast=__UpperCamelCase			)
							for tokenizer in tokenizers:
									with self.subTest(f'''{tokenizer.__class__.__name__}'''			):
											_snake_case     	=	"<unk>"
											_snake_case     	=	"<mask>"

											_snake_case     	=	tokenizer.tokenize(__UpperCamelCase			)
											_snake_case     	=	tokenizer.tokenize(__UpperCamelCase			)

											self.assertEqual(len(__UpperCamelCase			),  1			)
											self.assertEqual(len(__UpperCamelCase			),  1			)
											self.assertEqual(token_a[0],  __UpperCamelCase			)
											self.assertEqual(token_a[0],  __UpperCamelCase			)





