






"""simple docstring"""


import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VivitConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor


if is_torch_available():
       import torch
       from torch import nn

       from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VivitForVideoClassification, VivitModel
       from transformers.models.vivit.modeling_vivit import VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
       from transformers import VivitImageProcessor


class  snake_case		:


   '''simple docstring'''

   def __init__(     self			:					List[Any],	_lowerCamelCase			:					List[Any],	_lowerCamelCase			:					int=2,	_lowerCamelCase			:					Dict=True,	_lowerCamelCase			:					Any=True,	_lowerCamelCase			:					Any=10,	_lowerCamelCase			:					List[Any]=10,	_lowerCamelCase			:					Optional[int]=8,	_lowerCamelCase			:					Optional[int]=[2, 4, 4],	_lowerCamelCase			:					int=3,	_lowerCamelCase			:					Optional[int]=7_68,	_lowerCamelCase			:					str=5,	_lowerCamelCase			:					str=4,	_lowerCamelCase			:					Dict=37,	_lowerCamelCase			:					Tuple="gelu_fast",	_lowerCamelCase			:					Any=0.0,	_lowerCamelCase			:					Union[str, Any]=0.0,	_lowerCamelCase			:					Tuple=0.02,	_lowerCamelCase			:					Tuple=1e-06,	_lowerCamelCase			:					Union[str, Any]=True,	_lowerCamelCase			:					Union[str, Any]=None,	):

     '''simple docstring'''
     __A						      =			parent
     __A						      =			batch_size
     __A						      =			is_training
     __A						      =			use_labels
     __A						      =			num_labels
     __A						      =			image_size
     __A						      =			num_frames
     __A						      =			tubelet_size
     __A						      =			num_channels
     __A						      =			hidden_size
     __A						      =			num_hidden_layers
     __A						      =			num_attention_heads
     __A						      =			intermediate_size
     __A						      =			hidden_act
     __A						      =			hidden_dropout_prob
     __A						      =			attention_probs_dropout_prob
     __A						      =			initializer_range
     __A						      =			layer_norm_eps
     __A						      =			qkv_bias
     __A						      =			scope

     __A						      =			(
         (self.image_size // self.tubelet_size[2])
         * (self.image_size // self.tubelet_size[1])
         * (self.num_frames // self.tubelet_size[0])
     ) + 1  # CLS token


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			floats_tensor(
         [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]  )

     __A						      =			None
     if self.use_labels:
       __A						      =			ids_tensor([self.batch_size],	self.num_labels  )

     __A						      =			self.get_config()

     return config, pixel_values, labels


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			VivitConfig(
         num_frames=self.num_frames,	image_size=self.image_size,	tubelet_size=self.tubelet_size,	num_channels=self.num_channels,	hidden_size=self.hidden_size,	num_hidden_layers=self.num_hidden_layers,	num_attention_heads=self.num_attention_heads,	intermediate_size=self.intermediate_size,	hidden_act=self.hidden_act,	hidden_dropout_prob=self.hidden_dropout_prob,	attention_probs_dropout_prob=self.attention_probs_dropout_prob,	initializer_range=self.initializer_range,	layer_norm_eps=self.layer_norm_eps,	qkv_bias=self.qkv_bias,	)
     __A						      =			self.num_labels
     return config


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	_lowerCamelCase			:					Any,	_lowerCamelCase			:					List[str],	_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			VivitModel(config=_lowerCamelCase  )
     model.to(_lowerCamelCase  )
     model.eval()
     __A						      =			model(_lowerCamelCase  )
     self.parent.assertEqual(result.last_hidden_state.shape,	(self.batch_size, self.seq_length, self.hidden_size)  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					Tuple,	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					List[str]  ):

     '''simple docstring'''
     __A						      =			VivitForVideoClassification(_lowerCamelCase  )
     model.to(_lowerCamelCase  )
     model.eval()

     __A						      =			model(_lowerCamelCase  )

     # verify the logits shape
     __A						      =			torch.Size((self.batch_size, self.num_labels)  )
     self.parent.assertEqual(result.logits.shape,	_lowerCamelCase  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			self.prepare_config_and_inputs()
     __A      ,   __A      ,   __A						      =			config_and_inputs
     __A						      =			{'''pixel_values''': pixel_values}
     return config, inputs_dict





@require_torch
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     List[str]									=			(VivitModel, VivitForVideoClassification) if is_torch_available() else ()

   A_						:     Optional[Any]									=			False
   A_						:     List[str]									=			False
   A_						:     Optional[Any]									=			False
   A_						:     str									=			False


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			VivitModelTester(self  )
     __A						      =			ConfigTester(self,	config_class=_lowerCamelCase,	has_text_modality=_lowerCamelCase,	hidden_size=37  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					Optional[Any],	_lowerCamelCase			:					Optional[Any]=False  ):

     '''simple docstring'''
     __A						      =			copy.deepcopy(_lowerCamelCase  )

     if return_labels:
       if model_class in get_values(_lowerCamelCase  ):
         __A						      =			torch.zeros(
             self.model_tester.batch_size,	dtype=torch.long,	device=_lowerCamelCase  )

     return inputs_dict


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     self.config_tester.run_common_tests()


   @unittest.skip(reason='''Vivit does not use inputs_embeds'''  )
   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     pass


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       __A						      =			model_class(_lowerCamelCase  )
       self.assertIsInstance(model.get_input_embeddings(),	(nn.Module)  )
       __A						      =			model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(_lowerCamelCase,	nn.Linear  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       __A						      =			model_class(_lowerCamelCase  )
       __A						      =			inspect.signature(model.forward  )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       __A						      =			[*signature.parameters.keys()]

       __A						      =			['''pixel_values''', '''head_mask''']
       self.assertListEqual(arg_names[:2],	_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_video_classification(*_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     for model_name in VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       __A						      =			VivitModel.from_pretrained(_lowerCamelCase  )
       self.assertIsNotNone(_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     __A						      =			True

     for model_class in self.all_model_classes:
       __A						      =			self.model_tester.seq_length

       __A						      =			True
       __A						      =			False
       __A						      =			True
       __A						      =			model_class(_lowerCamelCase  )
       model.to(_lowerCamelCase  )
       model.eval()
       with torch.no_grad():
         __A						      =			model(**self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )  )
       __A						      =			outputs.attentions
       self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )

       # check that output_attentions also work using config
       del inputs_dict["output_attentions"]
       __A						      =			True
       __A						      =			model_class(_lowerCamelCase  )
       model.to(_lowerCamelCase  )
       model.eval()
       with torch.no_grad():
         __A						      =			model(**self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )  )
       __A						      =			outputs.attentions
       self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )

       self.assertListEqual(
           list(attentions[0].shape[-3:]  ),	[self.model_tester.num_attention_heads, seq_len, seq_len],	)
       __A						      =			len(_lowerCamelCase  )

       # Check attention is always last and order is fine
       __A						      =			True
       __A						      =			True
       __A						      =			model_class(_lowerCamelCase  )
       model.to(_lowerCamelCase  )
       model.eval()
       with torch.no_grad():
         __A						      =			model(**self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )  )

       self.assertEqual(out_len + 1,	len(_lowerCamelCase  )  )

       __A						      =			outputs.attentions

       self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )

       self.assertListEqual(
           list(self_attentions[0].shape[-3:]  ),	[self.model_tester.num_attention_heads, seq_len, seq_len],	)


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     def check_hidden_states_output(_lowerCamelCase			:					str,	_lowerCamelCase			:					Optional[Any],	_lowerCamelCase			:					Union[str, Any]  ):
       __A						      =			model_class(_lowerCamelCase  )
       model.to(_lowerCamelCase  )
       model.eval()

       with torch.no_grad():
         __A						      =			model(**self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )  )

       __A						      =			outputs.hidden_states
       __A						      =			self.model_tester.num_hidden_layers + 1
       self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )

       __A						      =			self.model_tester.seq_length

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]  ),	[seq_length, self.model_tester.hidden_size],	)

     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       __A						      =			True
       check_hidden_states_output(_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase  )

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       __A						      =			True

       check_hidden_states_output(_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase  )






   @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''  )
   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     pass





def  lowerCAmelCase    (   ):




  """simple docstring"""







  __A						      =			hf_hub_download(
      repo_id='''hf-internal-testing/spaghetti-video'''		,	filename='''eating_spaghetti_32_frames.npy'''		,	repo_type='''dataset'''			)
  __A						      =			np.load(__UpperCamelCase			)
  return list(__UpperCamelCase			)





@require_torch
@require_vision
class  snake_case		(							unittest.TestCase						):


   '''simple docstring'''

   @cached_property
   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     return VivitImageProcessor() if is_vision_available() else None






   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			VivitForVideoClassification.from_pretrained('''google/vivit-b-16x2-kinetics400'''  ).to(_lowerCamelCase  )

     __A						      =			self.default_image_processor
     __A						      =			prepare_video()
     __A						      =			image_processor(_lowerCamelCase,	return_tensors='''pt'''  ).to(_lowerCamelCase  )

     # forward pass
     with torch.no_grad():
       __A						      =			model(**_lowerCamelCase  )

     # verify the logits
     __A						      =			torch.Size((1, 4_00)  )
     self.assertEqual(outputs.logits.shape,	_lowerCamelCase  )

     # taken from original model
     __A						      =			torch.tensor([-1.05_43, 2.07_64, -0.21_04, 0.44_39, -0.96_58]  ).to(_lowerCamelCase  )

     self.assertTrue(torch.allclose(outputs.logits[0, :5],	_lowerCamelCase,	atol=1e-4  )  )












"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class  snake_case		(							_lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     str									=			ElectraTokenizer
   A_						:     List[Any]									=			ElectraTokenizerFast
   A_						:     Union[str, Any]									=			True
   A_						:     List[str]									=			True
   A_						:     Optional[Any]									=			filter_non_english


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     super().setUp()

     __A						      =			[
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     __A						      =			os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file,	'''w''',	encoding='''utf-8'''  ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple,	_lowerCamelCase			:					int  ):

     '''simple docstring'''
     __A						      =			'''UNwant\u00E9d,running'''
     __A						      =			'''unwanted, running'''
     return input_text, output_text


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class(self.vocab_file  )

     __A						      =			tokenizer.tokenize('''UNwant\u00E9d,running'''  )
     self.assertListEqual(_lowerCamelCase,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase  ),	[9, 6, 7, 12, 10, 11]  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     if not self.test_rust_tokenizer:
       return

     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     # With lower casing
     __A						      =			self.get_tokenizer(do_lower_case=_lowerCamelCase  )
     __A						      =			self.get_rust_tokenizer(do_lower_case=_lowerCamelCase  )

     __A						      =			'''UNwant\u00E9d,running'''

     __A						      =			tokenizer.tokenize(_lowerCamelCase  )
     __A						      =			rust_tokenizer.tokenize(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

     __A						      =			self.get_rust_tokenizer()
     __A						      =			tokenizer.encode(_lowerCamelCase  )
     __A						      =			rust_tokenizer.encode(_lowerCamelCase  )
     self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  ),	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''h\u00E9llo''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ),	['''hello''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	strip_accents=_lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ),	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			BasicTokenizer(do_lower_case=_lowerCamelCase,	never_split=['''[UNK]''']  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  ),	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

     __A						      =			{}
     for i, token in enumerate(_lowerCamelCase  ):
       __A						      =			i
     __A						      =			WordpieceTokenizer(vocab=_lowerCamelCase,	unk_token='''[UNK]'''  )

     self.assertListEqual(tokenizer.tokenize(''''''  ),	[]  )

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''  ),	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  ),	['''[UNK]''', '''runn''', '''##ing''']  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     self.assertTrue(_is_whitespace(''' '''  )  )
     self.assertTrue(_is_whitespace('''\t'''  )  )
     self.assertTrue(_is_whitespace('''\r'''  )  )
     self.assertTrue(_is_whitespace('''\n'''  )  )
     self.assertTrue(_is_whitespace('''\u00A0'''  )  )

     self.assertFalse(_is_whitespace('''A'''  )  )
     self.assertFalse(_is_whitespace('''-'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any  ):

     '''simple docstring'''
     self.assertTrue(_is_control('''\u0005'''  )  )

     self.assertFalse(_is_control('''A'''  )  )
     self.assertFalse(_is_control(''' '''  )  )
     self.assertFalse(_is_control('''\t'''  )  )
     self.assertFalse(_is_control('''\r'''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     self.assertTrue(_is_punctuation('''-'''  )  )
     self.assertTrue(_is_punctuation('''$'''  )  )
     self.assertTrue(_is_punctuation('''`'''  )  )
     self.assertTrue(_is_punctuation('''.'''  )  )

     self.assertFalse(_is_punctuation('''A'''  )  )
     self.assertFalse(_is_punctuation(''' '''  )  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.get_tokenizer()
     __A						      =			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )

     self.assertListEqual(
         [rust_tokenizer.tokenize(_lowerCamelCase  ) for t in ['''Test''', '''\xad''', '''test''']],	[['''[UNK]'''], [], ['''[UNK]''']]  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''  )

     __A						      =			tokenizer.encode('''sequence builders''',	add_special_tokens=_lowerCamelCase  )
     __A						      =			tokenizer.encode('''multi-sequence build''',	add_special_tokens=_lowerCamelCase  )

     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  )
     __A						      =			tokenizer.build_inputs_with_special_tokens(_lowerCamelCase,	_lowerCamelCase  )

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
         __A						      =			tokenizer_r.encode_plus(
             _lowerCamelCase,	return_attention_mask=_lowerCamelCase,	return_token_type_ids=_lowerCamelCase,	return_offsets_mapping=_lowerCamelCase,	add_special_tokens=_lowerCamelCase,	)

         __A						      =			tokenizer_r.do_lower_case if hasattr(_lowerCamelCase,	'''do_lower_case'''  ) else False
         __A						      =			(
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''A'''),
                 ((1, 2), ''','''),
                 ((3, 5), '''na'''),
                 ((5, 6), '''##ï'''),
                 ((6, 8), '''##ve'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''Allen'''),
                 ((21, 23), '''##NL'''),
                 ((23, 24), '''##P'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), '''a'''),
                 ((1, 2), ''','''),
                 ((3, 8), '''naive'''),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), '''allen'''),
                 ((21, 23), '''##nl'''),
                 ((23, 24), '''##p'''),
                 ((25, 33), '''sentence'''),
                 ((33, 34), '''.'''),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results],	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
         self.assertEqual([e[0] for e in expected_results],	tokens['''offset_mapping''']  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A						      =			['''的''', '''人''', '''有''']
     __A						      =			''''''.join(_lowerCamelCase  )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
         __A						      =			True
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )

         __A						      =			False
         __A						      =			self.rust_tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
         __A						      =			self.tokenizer_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )

         __A						      =			tokenizer_r.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )
         __A						      =			tokenizer_p.encode(_lowerCamelCase,	add_special_tokens=_lowerCamelCase  )

         __A						      =			tokenizer_r.convert_ids_to_tokens(_lowerCamelCase  )
         __A						      =			tokenizer_p.convert_ids_to_tokens(_lowerCamelCase  )

         # it is expected that only the first Chinese character is not preceded by "##".
         __A						      =			[
             f'##{token}' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase  )
         ]
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )
         self.assertListEqual(_lowerCamelCase,	_lowerCamelCase  )




