


'''simple docstring'''

import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase
@require_tokenizers
class   __lowerCAmelCase						(unittest.TestCase					):





						'''simple docstring'''






						lowerCAmelCase__ :					List[Any]   =							EsmTokenizer



						def 				UpperCamelCase__		(self   :   List[str]			):

													'''simple docstring'''







													super().setUp()
													lowercase__				      =			tempfile.mkdtemp()
													# fmt: off
													lowercase__				      =			['''<cls>''', '''<pad>''', '''<eos>''', '''<unk>''', '''L''', '''A''', '''G''', '''V''', '''S''', '''E''', '''R''', '''T''', '''I''', '''D''', '''P''', '''K''', '''Q''', '''N''', '''F''', '''Y''', '''M''', '''H''', '''W''', '''C''', '''X''', '''B''', '''U''', '''Z''', '''O''', '''.''', '''-''', '''<null_1>''', '''<mask>''']  # noqa: E501
													# fmt: on
													lowercase__				      =			os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['''vocab_file''']			)
													with open(self.vocab_file       ,						'''w'''       ,						encoding='''utf-8'''			) as vocab_writer:
																				vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)



						def 				UpperCamelCase__		(self   :   List[str]       ,						**UpperCamelCase   :   List[Any]			):

													'''simple docstring'''







													return [self.get_tokenizer(**lowerCAmelCase__			)]



						def 				UpperCamelCase__		(self   :   Any       ,						**UpperCamelCase   :   Any			):

													'''simple docstring'''







													return self.tokenizer_class.from_pretrained(self.tmpdirname       ,						**lowerCAmelCase__			)



						def 				UpperCamelCase__		(self   :   int			):

													'''simple docstring'''







													lowercase__				      =			self.tokenizer_class(self.vocab_file			)

													lowercase__				      =			tokenizer.tokenize('''LAGVS'''			)
													self.assertListEqual(lowerCAmelCase__       ,						['''L''', '''A''', '''G''', '''V''', '''S''']			)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)       ,						[4, 5, 6, 7, 8]			)



						def 				UpperCamelCase__		(self   :   str			):

													'''simple docstring'''







													lowercase__				      =			self.tokenizer_class(self.vocab_file			)

													lowercase__				      =			'''LAGVS'''
													self.assertListEqual(tokenizer.encode(lowerCAmelCase__			)       ,						[0, 4, 5, 6, 7, 8, 2]			)



						def 				UpperCamelCase__		(self   :   Optional[int]			):

													'''simple docstring'''







													lowercase__				      =			self.tokenizer_class(self.vocab_file			)

													lowercase__				      =			['''LAGVS''', '''WCB''']
													lowercase__				      =			tokenizer(lowerCAmelCase__       ,						padding=lowerCAmelCase__			)['''input_ids''']

													self.assertListEqual(lowerCAmelCase__       ,						[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]			)



						def 				UpperCamelCase__		(self   :   Optional[Any]			):

													'''simple docstring'''







													lowercase__				      =			self.tokenizer_class(self.vocab_file			)

													lowercase__				      =			['''LAGVS''', '''WCB''']
													lowercase__				      =			tokenizer(lowerCAmelCase__       ,						padding=lowerCAmelCase__			)['''input_ids''']

													self.assertListEqual(lowerCAmelCase__       ,						[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]			)






						def 				UpperCamelCase__		(self   :   Any			):

													'''simple docstring'''







													lowercase__				      =			self.get_tokenizers(fast=lowerCAmelCase__			)
													for tokenizer in tokenizers:
																				with self.subTest(f"{tokenizer.__class__.__name__}"			):
																											lowercase__				      =			'''<unk>'''
																											lowercase__				      =			'''<mask>'''

																											lowercase__				      =			tokenizer.tokenize(lowerCAmelCase__			)
																											lowercase__				      =			tokenizer.tokenize(lowerCAmelCase__			)

																											self.assertEqual(len(lowerCAmelCase__			)       ,						1			)
																											self.assertEqual(len(lowerCAmelCase__			)       ,						1			)
																											self.assertEqual(token_a[0]       ,						lowerCAmelCase__			)
																											self.assertEqual(token_a[0]       ,						lowerCAmelCase__			)








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase							:       str  						=     {'configuration_nat': ['NAT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NatConfig']}


try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:       Tuple  						=     [
     'NAT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'NatForImageClassification',
     'NatModel',
     'NatPreTrainedModel',
     'NatBackbone',
 ]

if TYPE_CHECKING:
 from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nat import (
      NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
      NatBackbone,
      NatForImageClassification,
      NatModel,
      NatPreTrainedModel,
  )

else:
 import sys

 lowerCamelCase							:       Optional[Any]  						=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




