





import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 					UpperCamelCase_    (       UpperCamelCase__			,					unittest.TestCase      ):
							lowerCamelCase_          =   HerbertTokenizer
							lowerCamelCase_          =   HerbertTokenizerFast
							lowerCamelCase_          =   True


							def        _snake_case					(							self      :str						)				->				str:



									"""simple docstring"""



									super().setUp()

									# Use a simpler test file without japanese/chinese characters
									with open(f'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''							,			encoding="""utf-8"""						) as f_data:
											SCREAMING_SNAKE_CASE__								= f_data.read().replace("""\n\n"""							,			"""\n"""						).strip()

									SCREAMING_SNAKE_CASE__								= [
									    """<s>""",
									    """</s>""",
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """w</w>""",
									    """r</w>""",
									    """t</w>""",
									    """lo""",
									    """low""",
									    """er</w>""",
									    """low</w>""",
									    """lowest</w>""",
									    """newer</w>""",
									    """wider</w>""",
									    """,</w>""",
									    """<unk>""",
									]
									SCREAMING_SNAKE_CASE__								= dict(zip(__A							,			range(len(__A						)						)						)						)
									SCREAMING_SNAKE_CASE__								= ["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

									SCREAMING_SNAKE_CASE__								= os.path.join(self.tmpdirname							,			VOCAB_FILES_NAMES["""vocab_file"""]						)
									SCREAMING_SNAKE_CASE__								= os.path.join(self.tmpdirname							,			VOCAB_FILES_NAMES["""merges_file"""]						)
									with open(self.vocab_file							,			"""w"""						) as fp:
											fp.write(json.dumps(__A						)						)
									with open(self.merges_file							,			"""w"""						) as fp:
											fp.write("""\n""".join(__A						)						)


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= """lower newer"""
									SCREAMING_SNAKE_CASE__								= """lower newer"""
									return input_text, output_text


							def        _snake_case					(							self      :Any						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.tokenizer_class(vocab_file=self.vocab_file							,			merges_file=self.merges_file						)

									SCREAMING_SNAKE_CASE__								= """lower"""
									SCREAMING_SNAKE_CASE__								= ["""low""", """er</w>"""]
									SCREAMING_SNAKE_CASE__								= tokenizer.tokenize(__A						)
									self.assertListEqual(__A							,			__A						)

									SCREAMING_SNAKE_CASE__								= tokens + ["""<unk>"""]
									SCREAMING_SNAKE_CASE__								= [16, 17, 23]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A						)							,			__A						)


							def        _snake_case					(							self      :Tuple						)				->				Optional[Any]:



									"""simple docstring"""



									if not self.test_rust_tokenizer:
											return

									SCREAMING_SNAKE_CASE__								= self.get_tokenizer()
									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()

									SCREAMING_SNAKE_CASE__								= """lower,newer"""

									SCREAMING_SNAKE_CASE__								= tokenizer.tokenize(__A						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.tokenize(__A						)
									self.assertListEqual(__A							,			__A						)

									SCREAMING_SNAKE_CASE__								= tokenizer.encode(__A							,			add_special_tokens=__A						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(__A							,			add_special_tokens=__A						)
									self.assertListEqual(__A							,			__A						)

									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE__								= tokenizer.encode(__A						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(__A						)
									self.assertListEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Optional[int]						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.tokenizer_class.from_pretrained("""allegro/herbert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.encode("""konstruowanie sekwencji"""							,			add_special_tokens=__A						)
									SCREAMING_SNAKE_CASE__								= tokenizer.encode("""konstruowanie wielu sekwencji"""							,			add_special_tokens=__A						)

									SCREAMING_SNAKE_CASE__								= tokenizer.build_inputs_with_special_tokens(__A						)
									SCREAMING_SNAKE_CASE__								= tokenizer.build_inputs_with_special_tokens(__A							,			__A						)

									assert encoded_sentence == [0] + text + [2]
									assert encoded_pair == [0] + text + [2] + text_a + [2]






import unittest

from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    PreTrainedTokenizer,
    is_vision_available,
)
from transformers.pipelines import ImageClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
	from PIL import Image
else:




	class 					UpperCamelCase_    :


								@staticmethod
								def        _snake_case					(							*__A      :Optional[Any]							,			**__A      :Any						)				->				str:



										"""simple docstring"""



										pass




@is_pipeline_test
@require_torch_or_tf
@require_vision
class 					UpperCamelCase_    (       unittest.TestCase      ):
							lowerCamelCase_          =   MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
							lowerCamelCase_          =   TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING


							def        _snake_case					(							self      :Union[str, Any]							,			__A      :str							,			__A      :Dict							,			__A      :List[str]						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= ImageClassificationPipeline(model=__A							,			image_processor=__A							,			top_k=2						)
									SCREAMING_SNAKE_CASE__								= [
									    Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						),
									    """http://images.cocodataset.org/val2017/000000039769.jpg""",
									]
									return image_classifier, examples


							def        _snake_case					(							self      :List[Any]							,			__A      :Any							,			__A      :List[str]						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= image_classifier("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						)

									self.assertEqual(
									    __A							,			[
									        {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        {"""score""": ANY(__A						), """label""": ANY(__A						)},
									    ]							,			)

									import datasets

									SCREAMING_SNAKE_CASE__								= datasets.load_dataset("""hf-internal-testing/fixtures_image_utils"""							,			"""image"""							,			split="""test"""						)

									# Accepts URL + PIL.Image + lists
									SCREAMING_SNAKE_CASE__								= image_classifier(
									    [
									        Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						),
									        """http://images.cocodataset.org/val2017/000000039769.jpg""",
									        # RGBA
									        dataset[0]["""file"""],
									        # LA
									        dataset[1]["""file"""],
									        # L
									        dataset[2]["""file"""],
									    ]						)
									self.assertEqual(
									    __A							,			[
									        [
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        ],
									        [
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        ],
									        [
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        ],
									        [
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        ],
									        [
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									            {"""score""": ANY(__A						), """label""": ANY(__A						)},
									        ],
									    ]							,			)


							@require_torch
							def        _snake_case					(							self      :str						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= """hf-internal-testing/tiny-random-vit"""
									SCREAMING_SNAKE_CASE__								= pipeline("""image-classification"""							,			model=__A						)

									SCREAMING_SNAKE_CASE__								= image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""						)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}]							,			)

									SCREAMING_SNAKE_CASE__								= image_classifier(
									    [
									        """http://images.cocodataset.org/val2017/000000039769.jpg""",
									        """http://images.cocodataset.org/val2017/000000039769.jpg""",
									    ]							,			top_k=2							,			)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[
									        [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
									        [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
									    ]							,			)


							@require_tf
							def        _snake_case					(							self      :Any						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= """hf-internal-testing/tiny-random-vit"""
									SCREAMING_SNAKE_CASE__								= pipeline("""image-classification"""							,			model=__A							,			framework="""tf"""						)

									SCREAMING_SNAKE_CASE__								= image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""						)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}]							,			)

									SCREAMING_SNAKE_CASE__								= image_classifier(
									    [
									        """http://images.cocodataset.org/val2017/000000039769.jpg""",
									        """http://images.cocodataset.org/val2017/000000039769.jpg""",
									    ]							,			top_k=2							,			)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[
									        [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
									        [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
									    ]							,			)


							def        _snake_case					(							self      :Dict						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= PreTrainedTokenizer()

									# Assert that the pipeline can be initialized with a feature extractor that is not in any mapping
									SCREAMING_SNAKE_CASE__								= pipeline(
									    """image-classification"""							,			model="""hf-internal-testing/tiny-random-vit"""							,			tokenizer=__A						)

									self.assertIs(image_classifier.tokenizer							,			__A						)


							@slow
							@require_torch
							def        _snake_case					(							self      :Optional[int]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pipeline("""image-classification"""							,			model="""deepmind/vision-perceiver-conv"""						)
									SCREAMING_SNAKE_CASE__								= image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""						)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[
									        {"""score""": 0.4_3_8_5, """label""": """tabby, tabby cat"""},
									        {"""score""": 0.3_2_1, """label""": """tiger cat"""},
									        {"""score""": 0.0_5_0_2, """label""": """Egyptian cat"""},
									        {"""score""": 0.0_1_3_7, """label""": """crib, cot"""},
									        {"""score""": 0.0_0_7, """label""": """radiator"""},
									    ]							,			)

									SCREAMING_SNAKE_CASE__								= pipeline("""image-classification"""							,			model="""deepmind/vision-perceiver-fourier"""						)
									SCREAMING_SNAKE_CASE__								= image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""						)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[
									        {"""score""": 0.5_6_5_8, """label""": """tabby, tabby cat"""},
									        {"""score""": 0.1_3_0_9, """label""": """tiger cat"""},
									        {"""score""": 0.0_7_2_2, """label""": """Egyptian cat"""},
									        {"""score""": 0.0_7_0_7, """label""": """remote control, remote"""},
									        {"""score""": 0.0_0_8_2, """label""": """computer keyboard, keypad"""},
									    ]							,			)

									SCREAMING_SNAKE_CASE__								= pipeline("""image-classification"""							,			model="""deepmind/vision-perceiver-learned"""						)
									SCREAMING_SNAKE_CASE__								= image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""						)
									self.assertEqual(
									    nested_simplify(__A							,			decimals=4						)							,			[
									        {"""score""": 0.3_0_2_2, """label""": """tabby, tabby cat"""},
									        {"""score""": 0.2_3_6_2, """label""": """Egyptian cat"""},
									        {"""score""": 0.1_8_5_6, """label""": """tiger cat"""},
									        {"""score""": 0.0_3_2_4, """label""": """remote control, remote"""},
									        {"""score""": 0.0_0_9_6, """label""": """quilt, comforter, comfort, puff"""},
									    ]							,			)