
'''simple docstring'''

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					a_					(A__							,     unittest.TestCase      ):
       __lowerCAmelCase      :			Optional[Any]      = ElectraTokenizer
       __lowerCAmelCase      :			Dict      = ElectraTokenizerFast
       __lowerCAmelCase      :			List[Any]      = True
       __lowerCAmelCase      :			int      = True
       __lowerCAmelCase      :			Union[str, Any]      = filter_non_english




       def 							__UpperCamelCase    (       self					):
         super().setUp()

         _lowerCAmelCase    :					Union[str, Any]							=   [
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         _lowerCAmelCase    :					int							=   os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["""vocab_file"""]					)
         with open(self.vocab_file				,				"""w"""				,				encoding="""utf-8"""					) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)




       def 							__UpperCamelCase    (       self				,				snake_case_					):
         _lowerCAmelCase    :					str							=   "UNwant\u00E9d,running"
         _lowerCAmelCase    :					Optional[int]							=   "unwanted, running"
         return input_text, output_text




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					int							=   self.tokenizer_class(self.vocab_file					)

         _lowerCAmelCase    :					List[Any]							=   tokenizer.tokenize("""UNwant\u00E9d,running"""					)
         self.assertListEqual(_A				,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]					)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_A					)				,				[9, 6, 7, 1_2, 1_0, 1_1]					)




       def 							__UpperCamelCase    (       self					):






         if not self.test_rust_tokenizer:
           return

         _lowerCAmelCase    :					Any							=   self.get_tokenizer()
         _lowerCAmelCase    :					int							=   self.get_rust_tokenizer()

         _lowerCAmelCase    :					Dict							=   "UNwant\u00E9d,running"

         _lowerCAmelCase    :					Dict							=   tokenizer.tokenize(_A					)
         _lowerCAmelCase    :					List[str]							=   rust_tokenizer.tokenize(_A					)
         self.assertListEqual(_A				,				_A					)

         _lowerCAmelCase    :					List[str]							=   tokenizer.encode(_A				,				add_special_tokens=_A					)
         _lowerCAmelCase    :					str							=   rust_tokenizer.encode(_A				,				add_special_tokens=_A					)
         self.assertListEqual(_A				,				_A					)

         _lowerCAmelCase    :					Dict							=   self.get_rust_tokenizer()
         _lowerCAmelCase    :					Tuple							=   tokenizer.encode(_A					)
         _lowerCAmelCase    :					Tuple							=   rust_tokenizer.encode(_A					)
         self.assertListEqual(_A				,				_A					)

         # With lower casing
         _lowerCAmelCase    :					Tuple							=   self.get_tokenizer(do_lower_case=_A					)
         _lowerCAmelCase    :					Union[str, Any]							=   self.get_rust_tokenizer(do_lower_case=_A					)

         _lowerCAmelCase    :					Tuple							=   "UNwant\u00E9d,running"

         _lowerCAmelCase    :					Any							=   tokenizer.tokenize(_A					)
         _lowerCAmelCase    :					Optional[int]							=   rust_tokenizer.tokenize(_A					)
         self.assertListEqual(_A				,				_A					)

         _lowerCAmelCase    :					Dict							=   tokenizer.encode(_A				,				add_special_tokens=_A					)
         _lowerCAmelCase    :					List[str]							=   rust_tokenizer.encode(_A				,				add_special_tokens=_A					)
         self.assertListEqual(_A				,				_A					)

         _lowerCAmelCase    :					List[Any]							=   self.get_rust_tokenizer()
         _lowerCAmelCase    :					Union[str, Any]							=   tokenizer.encode(_A					)
         _lowerCAmelCase    :					Optional[int]							=   rust_tokenizer.encode(_A					)
         self.assertListEqual(_A				,				_A					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[Any]							=   BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""					)				,				["""ah""", """\u535A""", """\u63A8""", """zz"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[Any]							=   BasicTokenizer(do_lower_case=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)				,				["""hello""", """!""", """how""", """are""", """you""", """?"""]					)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)				,				["""hello"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[int]							=   BasicTokenizer(do_lower_case=_A				,				strip_accents=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)				,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]					)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)				,				["""h\u00E9llo"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Any							=   BasicTokenizer(do_lower_case=_A				,				strip_accents=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)				,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)				,				["""hello"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					int							=   BasicTokenizer(do_lower_case=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)				,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)				,				["""hello"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Tuple							=   BasicTokenizer(do_lower_case=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)				,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Tuple							=   BasicTokenizer(do_lower_case=_A				,				strip_accents=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)				,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[Any]							=   BasicTokenizer(do_lower_case=_A				,				strip_accents=_A					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)				,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Any							=   BasicTokenizer(do_lower_case=_A				,				never_split=["""[UNK]"""]					)

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""					)				,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Union[str, Any]							=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         _lowerCAmelCase    :					int							=   {}
         for i, token in enumerate(_A					):
           _lowerCAmelCase    :					str							=   i
         _lowerCAmelCase    :					Tuple							=   WordpieceTokenizer(vocab=_A				,				unk_token="""[UNK]"""					)

         self.assertListEqual(tokenizer.tokenize(""""""					)				,				[]					)

         self.assertListEqual(tokenizer.tokenize("""unwanted running"""					)				,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]					)

         self.assertListEqual(tokenizer.tokenize("""unwantedX running"""					)				,				["""[UNK]""", """runn""", """##ing"""]					)




       def 							__UpperCamelCase    (       self					):
         self.assertTrue(_is_whitespace(""" """					)					)
         self.assertTrue(_is_whitespace("""\t"""					)					)
         self.assertTrue(_is_whitespace("""\r"""					)					)
         self.assertTrue(_is_whitespace("""\n"""					)					)
         self.assertTrue(_is_whitespace("""\u00A0"""					)					)

         self.assertFalse(_is_whitespace("""A"""					)					)
         self.assertFalse(_is_whitespace("""-"""					)					)




       def 							__UpperCamelCase    (       self					):
         self.assertTrue(_is_control("""\u0005"""					)					)

         self.assertFalse(_is_control("""A"""					)					)
         self.assertFalse(_is_control(""" """					)					)
         self.assertFalse(_is_control("""\t"""					)					)
         self.assertFalse(_is_control("""\r"""					)					)




       def 							__UpperCamelCase    (       self					):
         self.assertTrue(_is_punctuation("""-"""					)					)
         self.assertTrue(_is_punctuation("""$"""					)					)
         self.assertTrue(_is_punctuation("""`"""					)					)
         self.assertTrue(_is_punctuation("""."""					)					)

         self.assertFalse(_is_punctuation("""A"""					)					)
         self.assertFalse(_is_punctuation(""" """					)					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					int							=   self.get_tokenizer()
         _lowerCAmelCase    :					Tuple							=   self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(_A					) for t in ["""Test""", """\xad""", """test"""]]				,				[["""[UNK]"""], [], ["""[UNK]"""]]					)

         self.assertListEqual(
             [rust_tokenizer.tokenize(_A					) for t in ["""Test""", """\xad""", """test"""]]				,				[["""[UNK]"""], [], ["""[UNK]"""]]					)




       @slow
       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[Any]							=   self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""					)

         _lowerCAmelCase    :					Dict							=   tokenizer.encode("""sequence builders"""				,				add_special_tokens=_A					)
         _lowerCAmelCase    :					Any							=   tokenizer.encode("""multi-sequence build"""				,				add_special_tokens=_A					)

         _lowerCAmelCase    :					Optional[Any]							=   tokenizer.build_inputs_with_special_tokens(_A					)
         _lowerCAmelCase    :					Any							=   tokenizer.build_inputs_with_special_tokens(_A				,				_A					)

         assert encoded_sentence == [1_0_1] + text + [1_0_2]
         assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




       def 							__UpperCamelCase    (       self					):






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
             _lowerCAmelCase    :					Tuple							=   self.rust_tokenizer_class.from_pretrained(_A				,				**_A					)

             _lowerCAmelCase    :					str							=   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
             _lowerCAmelCase    :					Dict							=   tokenizer_r.encode_plus(
                 _A				,				return_attention_mask=_A				,				return_token_type_ids=_A				,				return_offsets_mapping=_A				,				add_special_tokens=_A				,				)

             _lowerCAmelCase    :					Dict							=   tokenizer_r.do_lower_case if hasattr(_A				,				"""do_lower_case"""					) else False
             _lowerCAmelCase    :					List[Any]							=   (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]				,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]					)					)
             self.assertEqual([e[0] for e in expected_results]				,				tokens["""offset_mapping"""]					)







       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					str							=   ["的", "人", "有"]
         _lowerCAmelCase    :					Dict							=   "".join(_A					)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
             _lowerCAmelCase    :					str							=   True
             _lowerCAmelCase    :					Dict							=   self.tokenizer_class.from_pretrained(_A				,				**_A					)
             _lowerCAmelCase    :					Optional[int]							=   self.rust_tokenizer_class.from_pretrained(_A				,				**_A					)

             _lowerCAmelCase    :					Tuple							=   tokenizer_p.encode(_A				,				add_special_tokens=_A					)
             _lowerCAmelCase    :					List[Any]							=   tokenizer_r.encode(_A				,				add_special_tokens=_A					)

             _lowerCAmelCase    :					List[Any]							=   tokenizer_r.convert_ids_to_tokens(_A					)
             _lowerCAmelCase    :					int							=   tokenizer_p.convert_ids_to_tokens(_A					)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_A				,				_A					)
             self.assertListEqual(_A				,				_A					)

             _lowerCAmelCase    :					str							=   False
             _lowerCAmelCase    :					List[str]							=   self.rust_tokenizer_class.from_pretrained(_A				,				**_A					)
             _lowerCAmelCase    :					List[str]							=   self.tokenizer_class.from_pretrained(_A				,				**_A					)

             _lowerCAmelCase    :					Any							=   tokenizer_r.encode(_A				,				add_special_tokens=_A					)
             _lowerCAmelCase    :					Optional[Any]							=   tokenizer_p.encode(_A				,				add_special_tokens=_A					)

             _lowerCAmelCase    :					str							=   tokenizer_r.convert_ids_to_tokens(_A					)
             _lowerCAmelCase    :					Dict							=   tokenizer_p.convert_ids_to_tokens(_A					)

             # it is expected that only the first Chinese character is not preceded by "##".
             _lowerCAmelCase    :					int							=   [
                 f'##{token}' if idx != 0 else token for idx, token in enumerate(_A					)
             ]
             self.assertListEqual(_A				,				_A					)
             self.assertListEqual(_A				,				_A					)




'''simple docstring'''

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import ConvNextImageProcessor


class 					a_					(unittest.TestCase      ):




       def __init__(       self				,				snake_case_				,				snake_case_=7				,				snake_case_=3				,				snake_case_=1_8				,				snake_case_=3_0				,				snake_case_=4_0_0				,				snake_case_=True				,				snake_case_=None				,				snake_case_=0.875				,				snake_case_=True				,				snake_case_=[0.5, 0.5, 0.5]				,				snake_case_=[0.5, 0.5, 0.5]				,				):
         _lowerCAmelCase    :					List[str]							=   size if size is not None else {"""shortest_edge""": 2_0}
         _lowerCAmelCase    :					Tuple							=   parent
         _lowerCAmelCase    :					List[str]							=   batch_size
         _lowerCAmelCase    :					List[str]							=   num_channels
         _lowerCAmelCase    :					Tuple							=   image_size
         _lowerCAmelCase    :					Union[str, Any]							=   min_resolution
         _lowerCAmelCase    :					Tuple							=   max_resolution
         _lowerCAmelCase    :					List[Any]							=   do_resize
         _lowerCAmelCase    :					Dict							=   size
         _lowerCAmelCase    :					str							=   crop_pct
         _lowerCAmelCase    :					List[str]							=   do_normalize
         _lowerCAmelCase    :					Any							=   image_mean
         _lowerCAmelCase    :					Any							=   image_std







       def 							__UpperCamelCase    (       self					):
         return {
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_normalize": self.do_normalize,
             "do_resize": self.do_resize,
             "size": self.size,
             "crop_pct": self.crop_pct,
         }



@require_torch
@require_vision
class 					a_					(_a							,     unittest.TestCase      ):
       __lowerCAmelCase      :			Dict      = ConvNextImageProcessor if is_vision_available() else None




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					int							=   ConvNextImageProcessingTester(self					)




       @property
       def 							__UpperCamelCase    (       self					):
         return self.image_processor_tester.prepare_image_processor_dict()




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					List[Any]							=   self.image_processing_class(**self.image_processor_dict					)
         self.assertTrue(hasattr(snake_case_				,				"""do_resize"""					)					)
         self.assertTrue(hasattr(snake_case_				,				"""size"""					)					)
         self.assertTrue(hasattr(snake_case_				,				"""crop_pct"""					)					)
         self.assertTrue(hasattr(snake_case_				,				"""do_normalize"""					)					)
         self.assertTrue(hasattr(snake_case_				,				"""image_mean"""					)					)
         self.assertTrue(hasattr(snake_case_				,				"""image_std"""					)					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Any							=   self.image_processing_class.from_dict(self.image_processor_dict					)
         self.assertEqual(image_processor.size				,				{"""shortest_edge""": 2_0}					)

         _lowerCAmelCase    :					List[Any]							=   self.image_processing_class.from_dict(self.image_processor_dict				,				size=4_2					)
         self.assertEqual(image_processor.size				,				{"""shortest_edge""": 4_2}					)




       def 							__UpperCamelCase    (       self					):
         pass




       def 							__UpperCamelCase    (       self					):
         # Initialize image_processing
         _lowerCAmelCase    :					str							=   self.image_processing_class(**self.image_processor_dict					)
         # create random PIL images
         _lowerCAmelCase    :					str							=   prepare_image_inputs(self.image_processor_tester				,				equal_resolution=snake_case_					)
         for image in image_inputs:
           self.assertIsInstance(snake_case_				,				Image.Image					)

         # Test not batched input
         _lowerCAmelCase    :					Optional[int]							=   image_processing(image_inputs[0]				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)

         # Test batched
         _lowerCAmelCase    :					Optional[Any]							=   image_processing(snake_case_				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)




       def 							__UpperCamelCase    (       self					):
         # Initialize image_processing
         _lowerCAmelCase    :					int							=   self.image_processing_class(**self.image_processor_dict					)
         # create random numpy tensors
         _lowerCAmelCase    :					Any							=   prepare_image_inputs(self.image_processor_tester				,				equal_resolution=snake_case_				,				numpify=snake_case_					)
         for image in image_inputs:
           self.assertIsInstance(snake_case_				,				np.ndarray					)

         # Test not batched input
         _lowerCAmelCase    :					Optional[Any]							=   image_processing(image_inputs[0]				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)

         # Test batched
         _lowerCAmelCase    :					List[str]							=   image_processing(snake_case_				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)







       def 							__UpperCamelCase    (       self					):
         # Initialize image_processing
         _lowerCAmelCase    :					Tuple							=   self.image_processing_class(**self.image_processor_dict					)
         # create random PyTorch tensors
         _lowerCAmelCase    :					List[str]							=   prepare_image_inputs(self.image_processor_tester				,				equal_resolution=snake_case_				,				torchify=snake_case_					)
         for image in image_inputs:
           self.assertIsInstance(snake_case_				,				torch.Tensor					)

         # Test not batched input
         _lowerCAmelCase    :					List[str]							=   image_processing(image_inputs[0]				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)

         # Test batched
         _lowerCAmelCase    :					Optional[int]							=   image_processing(snake_case_				,				return_tensors="""pt"""					).pixel_values
         self.assertEqual(
             encoded_images.shape				,				(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["""shortest_edge"""],
                 self.image_processor_tester.size["""shortest_edge"""],
             )				,				)


