






"""simple docstring"""


from collections import OrderedDict
from dataclasses import fields
from typing import Any, Tuple

import numpy as np

from .import_utils import is_torch_available



def  lowerCAmelCase    (   __UpperCamelCase			):




  """simple docstring"""







  if is_torch_available():
    import torch

    if isinstance(__UpperCamelCase		,	torch.Tensor			):
      return True

  return isinstance(__UpperCamelCase		,	np.ndarray			)





class  snake_case		(							_lowerCAmelCase						):


   '''simple docstring'''

   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			fields(self  )

     # Safety and consistency checks
     if not len(_lowerCamelCase  ):
       raise ValueError(f'{self.__class__.__name__} has no fields.'  )

     __A						      =			getattr(self,	class_fields[0].name  )
     __A						      =			all(getattr(self,	field.name  ) is None for field in class_fields[1:]  )

     if other_fields_are_none and isinstance(_lowerCamelCase,	_lowerCamelCase  ):
       for key, value in first_field.items():
         __A						      =			value
     else:
       for field in class_fields:
         __A						      =			getattr(self,	field.name  )
         if v is not None:
           __A						      =			v


   def __delitem__(     self			:					Dict,	*_lowerCamelCase			:					int,	**_lowerCamelCase			:					Any  ):

     '''simple docstring'''
     raise Exception(f'You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.'  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	*_lowerCamelCase			:					List[str],	**_lowerCamelCase			:					str  ):

     '''simple docstring'''
     raise Exception(f'You cannot use ``setdefault`` on a {self.__class__.__name__} instance.'  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	*_lowerCamelCase			:					Optional[Any],	**_lowerCamelCase			:					str  ):

     '''simple docstring'''
     raise Exception(f'You cannot use ``pop`` on a {self.__class__.__name__} instance.'  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	*_lowerCamelCase			:					Any,	**_lowerCamelCase			:					Optional[int]  ):

     '''simple docstring'''
     raise Exception(f'You cannot use ``update`` on a {self.__class__.__name__} instance.'  )


   def __getitem__(     self			:					int,	_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     if isinstance(_lowerCamelCase,	_lowerCamelCase  ):
       __A						      =			dict(self.items()  )
       return inner_dict[k]
     else:
       return self.to_tuple()[k]


   def __setattr__(     self			:					Optional[int],	_lowerCamelCase			:					Tuple,	_lowerCamelCase			:					Union[str, Any]  ):

     '''simple docstring'''
     if name in self.keys() and value is not None:
       # Don't call self.__setitem__ to avoid recursion errors
       super().__setitem__(_lowerCamelCase,	_lowerCamelCase  )
     super().__setattr__(_lowerCamelCase,	_lowerCamelCase  )


   def __setitem__(     self			:					List[str],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					str  ):

     '''simple docstring'''
     # Will raise a KeyException if needed
     super().__setitem__(_lowerCamelCase,	_lowerCamelCase  )
     # Don't call self.__setattr__ to avoid recursion errors
     super().__setattr__(_lowerCamelCase,	_lowerCamelCase  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     return tuple(self[k] for k in self.keys()  )












"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     str									=			ElectraTokenizer
   A_						:     List[Any]									=			ElectraTokenizerFast
   A_						:     Union[str, Any]									=			True
   A_						:     List[str]									=			True
   A_						:     Optional[Any]									=			filter_non_english


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     super().setUp()

     __A						      =			[
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     __A						      =			os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file,	'''w''',	encoding='''utf-8'''  ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple,	_lowerCamelCase			:					int  ):

     '''simple docstring'''
     __A						      =			'''UNwant\u00E9d,running'''
     __A						      =			'''unwanted, running'''
     return input_text, output_text


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class(self.vocab_file  )

     __A						      =			tokenizer.tokenize('''UNwant\u00E9d,running'''  )
     self.assertListEqual(_lowerCamelCase,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase  ),	[9, 6, 7, 12, 10, 11]  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     if not self.test_rust_tokenizer:
       return

     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     # With lower casing
     __A						      =			self.get_tokenizer(do_lower_case=_lowerCamelCase  )
     __A						      =			self.get_rust_tokenizer(do_lower_case=_lowerCamelCase  )

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  ),	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''h\u00E9llo''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	never_split=['''[UNK]''']  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

     __A						      =			{}
     for i, token in enumerate(_lowerCamelCase  ):
       __A						      =			i
     __A						      =			WordpieceTokenizer(vocab=_lowerCamelCase,	unk_token='''[UNK]'''  )

     self.assertListEqual(tokenizer.tokenize(''''''  ),	[]  )

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''  ),	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  ),	['''[UNK]''', '''runn''', '''##ing''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     self.assertTrue(_is_whitespace(''' '''  )  )
     self.assertTrue(_is_whitespace('''\t'''  )  )
     self.assertTrue(_is_whitespace('''\r'''  )  )
     self.assertTrue(_is_whitespace('''\n'''  )  )
     self.assertTrue(_is_whitespace('''\u00A0'''  )  )

     self.assertFalse(_is_whitespace('''A'''  )  )
     self.assertFalse(_is_whitespace('''-'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     self.assertTrue(_is_control('''\u0005'''  )  )

     self.assertFalse(_is_control('''A'''  )  )
     self.assertFalse(_is_control(''' '''  )  )
     self.assertFalse(_is_control('''\t'''  )  )
     self.assertFalse(_is_control('''\r'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     self.assertTrue(_is_punctuation('''-'''  )  )
     self.assertTrue(_is_punctuation('''$'''  )  )
     self.assertTrue(_is_punctuation('''`'''  )  )
     self.assertTrue(_is_punctuation('''.'''  )  )

     self.assertFalse(_is_punctuation('''A'''  )  )
     self.assertFalse(_is_punctuation(''' '''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )

     self.assertListEqual(
         [rust_tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''  )

     __A						      =			tokenizer.encode('''sequence builders''',	add_special_tokens=_lowerCamelCase  )
     __A						      =			tokenizer.encode('''multi-sequence build''',	add_special_tokens=_lowerCamelCase  )

     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  )
     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase,	_lowerCamelCase  )

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
         __A						      =			tokenizer_r.encode_plus(
             _lowerCamelCase,	return_attention_mask=_lowerCamelCase,	return_token_type_ids=_lowerCamelCase,	return_offsets_mapping=_lowerCamelCase,	add_special_tokens=_lowerCamelCase,	)

         __A						      =			tokenizer_r.do_lower_case if hasattr(_lowerCamelCase,	'''do_lower_case'''  ) else False
         __A						      =			(
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''A'''),
                 ((1, 2), ''','''),
                 ((3, 5), '''na'''),
                 ((5, 6), '''##ï'''),
                 ((6, 8), '''##ve'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''Allen'''),
                 ((21, 23), '''##NL'''),
                 ((23, 24), '''##P'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''a'''),
                 ((1, 2), ''','''),
                 ((3, 8), '''naive'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''allen'''),
                 ((21, 23), '''##nl'''),
                 ((23, 24), '''##p'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results],	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
         self.assertEqual([e[0] for e in expected_results],	tokens['''offset_mapping''']  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			['''的''', '''人''', '''有''']
     __A						      =			''''''.join(_lowerCamelCase  )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			True
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

         __A						      =			False
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that only the first Chinese character is not preceded by "##".
         __A						      =			[
             f'##{token}' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase  )
         ]
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )




