

'''simple docstring'''






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import MobileNetVaImageProcessor
class      SCREAMING_SNAKE_CASE__      (    unittest.TestCase				):




    def __init__( self    ,					lowercase__    ,					lowercase__=7    ,					lowercase__=3    ,					lowercase__=18    ,					lowercase__=30    ,					lowercase__=400    ,					lowercase__=True    ,					lowercase__=None    ,					lowercase__=True    ,					lowercase__=None    ,					):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      List[Any]					=	size if size is not None else {"shortest_edge": 20}
          SCREAMING_SNAKE_CASE_				:      int					=	crop_size if crop_size is not None else {"height": 18, "width": 18}
          SCREAMING_SNAKE_CASE_				:      str					=	parent
          SCREAMING_SNAKE_CASE_				:      int					=	batch_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	num_channels
          SCREAMING_SNAKE_CASE_				:      int					=	image_size
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	min_resolution
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	max_resolution
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	do_resize
          SCREAMING_SNAKE_CASE_				:      Dict					=	size
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	do_center_crop
          SCREAMING_SNAKE_CASE_				:      str					=	crop_size







    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return {
              "do_resize": self.do_resize,
              "size": self.size,
              "do_center_crop": self.do_center_crop,
              "crop_size": self.crop_size,
          }






@require_torch
@require_vision
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							MobileNetVaImageProcessor if is_vision_available() else None




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	MobileNetVaImageProcessingTester(self      )




    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return self.image_processor_tester.prepare_image_processor_dict()




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	self.image_processing_class(**self.image_processor_dict      )
          self.assertTrue(hasattr(lowercase__    ,					"do_resize"      )      )
          self.assertTrue(hasattr(lowercase__    ,					"size"      )      )
          self.assertTrue(hasattr(lowercase__    ,					"do_center_crop"      )      )
          self.assertTrue(hasattr(lowercase__    ,					"center_crop"      )      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	self.image_processing_class.from_dict(self.image_processor_dict      )
          self.assertEqual(image_processor.size    ,					{"shortest_edge": 20}      )
          self.assertEqual(image_processor.crop_size    ,					{"height": 18, "width": 18}      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	self.image_processing_class.from_dict(self.image_processor_dict    ,					size=42    ,					crop_size=84      )
          self.assertEqual(image_processor.size    ,					{"shortest_edge": 42}      )
          self.assertEqual(image_processor.crop_size    ,					{"height": 84, "width": 84}      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          pass




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	self.image_processing_class(**self.image_processor_dict      )
          # create random PIL images
          SCREAMING_SNAKE_CASE_				:      Dict					=	prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=lowercase__      )
          for image in image_inputs:
                self.assertIsInstance(lowercase__    ,					Image.Image      )

          # Test not batched input
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	image_processing(image_inputs[0]    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  1,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)

          # Test batched
          SCREAMING_SNAKE_CASE_				:      Tuple					=	image_processing(lowercase__    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  self.image_processor_tester.batch_size,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Tuple					=	self.image_processing_class(**self.image_processor_dict      )
          # create random numpy tensors
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=lowercase__    ,					numpify=lowercase__      )
          for image in image_inputs:
                self.assertIsInstance(lowercase__    ,					np.ndarray      )

          # Test not batched input
          SCREAMING_SNAKE_CASE_				:      Tuple					=	image_processing(image_inputs[0]    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  1,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)

          # Test batched
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	image_processing(lowercase__    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  self.image_processor_tester.batch_size,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)







    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.image_processing_class(**self.image_processor_dict      )
          # create random PyTorch tensors
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=lowercase__    ,					torchify=lowercase__      )
          for image in image_inputs:
                self.assertIsInstance(lowercase__    ,					torch.Tensor      )

          # Test not batched input
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	image_processing(image_inputs[0]    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  1,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)

          # Test batched
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	image_processing(lowercase__    ,					return_tensors="pt"      ).pixel_values
          self.assertEqual(
              encoded_images.shape    ,					(
                  self.image_processor_tester.batch_size,
                  self.image_processor_tester.num_channels,
                  self.image_processor_tester.crop_size["height"],
                  self.image_processor_tester.crop_size["width"],
              )    ,					)




'''simple docstring'''






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							HerbertTokenizer
    _A         =							HerbertTokenizerFast
    _A         =							True




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"    ,					encoding="utf-8"      ) as f_data:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	f_data.read().replace("\n\n"    ,					"\n"      ).strip()

          SCREAMING_SNAKE_CASE_				:      Dict					=	[
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	dict(zip(lowercase__    ,					range(len(lowercase__      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

          SCREAMING_SNAKE_CASE_				:      Any					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["merges_file"]      )
          with open(self.vocab_file    ,					"w"      ) as fp:
                fp.write(json.dumps(lowercase__      )      )
          with open(self.merges_file    ,					"w"      ) as fp:
                fp.write("\n".join(lowercase__      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	"lower newer"
          SCREAMING_SNAKE_CASE_				:      str					=	"lower newer"
          return input_text, output_text




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.tokenizer_class(vocab_file=self.vocab_file    ,					merges_file=self.merges_file      )

          SCREAMING_SNAKE_CASE_				:      int					=	"lower"
          SCREAMING_SNAKE_CASE_				:      List[str]					=	["low", "er</w>"]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokens + ["<unk>"]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__      )    ,					lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if not self.test_rust_tokenizer:
                return

          SCREAMING_SNAKE_CASE_				:      Dict					=	self.get_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	"lower,newer"

          SCREAMING_SNAKE_CASE_				:      int					=	tokenizer.tokenize(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	rust_tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	rust_tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.encode(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	rust_tokenizer.encode(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )







    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode("konstruowanie sekwencji"    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	tokenizer.encode("konstruowanie wielu sekwencji"    ,					add_special_tokens=lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.build_inputs_with_special_tokens(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.build_inputs_with_special_tokens(lowercase__    ,					lowercase__      )

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]

