import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					import PIL

					from transformers import FlavaImageProcessor
					from transformers.image_utils import PILImageResampling
					from transformers.models.flava.image_processing_flava import (
					    FLAVA_CODEBOOK_MEAN,
					    FLAVA_CODEBOOK_STD,
					    FLAVA_IMAGE_MEAN,
					    FLAVA_IMAGE_STD,
					)
else:
					UpperCamelCase      			=     UpperCamelCase      			=     UpperCamelCase      			=     UpperCamelCase      			=     None


class _a       (	unittest.TestCase ):

	'''simple docstring'''



	def __init__(       self							,			__UpperCAmelCase							,			__UpperCAmelCase=7							,			__UpperCAmelCase=3							,			__UpperCAmelCase=30							,			__UpperCAmelCase=400							,			__UpperCAmelCase=True							,			__UpperCAmelCase=None							,			__UpperCAmelCase=True							,			__UpperCAmelCase=None							,			__UpperCAmelCase=None							,			__UpperCAmelCase=True							,			__UpperCAmelCase=1 / 255							,			__UpperCAmelCase=True							,			__UpperCAmelCase=FLAVA_IMAGE_MEAN							,			__UpperCAmelCase=FLAVA_IMAGE_STD							,			__UpperCAmelCase=14							,			__UpperCAmelCase=75							,			__UpperCAmelCase=None							,			__UpperCAmelCase=16							,			__UpperCAmelCase=0.3							,			__UpperCAmelCase=None							,			__UpperCAmelCase=True							,			__UpperCAmelCase=None							,			__UpperCAmelCase=None							,			__UpperCAmelCase=True							,			__UpperCAmelCase=None							,			__UpperCAmelCase=True							,			__UpperCAmelCase=True							,			__UpperCAmelCase=FLAVA_CODEBOOK_MEAN							,			__UpperCAmelCase=FLAVA_CODEBOOK_STD							,			):
			__A							:   int	  =				size if size is not None else {"height": 224, "width": 224}
			__A							:   Any	  =				crop_size if crop_size is not None else {"height": 224, "width": 224}
			__A							:   Any	  =				codebook_size if codebook_size is not None else {"height": 112, "width": 112}
			__A							:   Optional[Any]	  =				codebook_crop_size if codebook_crop_size is not None else {"height": 112, "width": 112}

			__A							:   Dict	  =				parent
			__A							:   List[Any]	  =				batch_size
			__A							:   Optional[int]	  =				num_channels
			__A							:   Dict	  =				do_resize
			__A							:   str	  =				do_rescale
			__A							:   List[str]	  =				rescale_factor
			__A							:   int	  =				min_resolution
			__A							:   Dict	  =				max_resolution
			__A							:   Union[str, Any]	  =				size
			__A							:   Dict	  =				resample if resample is not None else PILImageResampling.BICUBIC
			__A							:   Dict	  =				do_normalize
			__A							:   Optional[int]	  =				image_mean
			__A							:   Tuple	  =				image_std
			__A							:   Optional[int]	  =				do_center_crop
			__A							:   Optional[int]	  =				crop_size

			__A							:   Dict	  =				input_size_patches
			__A							:   Tuple	  =				total_mask_patches
			__A							:   int	  =				mask_group_max_patches
			__A							:   List[str]	  =				mask_group_min_patches
			__A							:   Union[str, Any]	  =				mask_group_min_aspect_ratio
			__A							:   Union[str, Any]	  =				mask_group_max_aspect_ratio

			__A							:   Optional[int]	  =				codebook_do_resize
			__A							:   Union[str, Any]	  =				codebook_size
			__A							:   List[str]	  =				codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
			__A							:   Union[str, Any]	  =				codebook_do_center_crop
			__A							:   Tuple	  =				codebook_crop_size
			__A							:   Optional[int]	  =				codebook_do_map_pixels
			__A							:   Union[str, Any]	  =				codebook_do_normalize
			__A							:   Any	  =				codebook_image_mean
			__A							:   Dict	  =				codebook_image_std

	def 					__UpperCAmelCase(       self   ):
			return {
			    "image_mean": self.image_mean,
			    "image_std": self.image_std,
			    "do_normalize": self.do_normalize,
			    "do_resize": self.do_resize,
			    "size": self.size,
			    "resample": self.resample,
			    "do_rescale": self.do_rescale,
			    "rescale_factor": self.rescale_factor,
			    "do_center_crop": self.do_center_crop,
			    "crop_size": self.crop_size,
			    "input_size_patches": self.input_size_patches,
			    "total_mask_patches": self.total_mask_patches,
			    "mask_group_max_patches": self.mask_group_max_patches,
			    "mask_group_min_patches": self.mask_group_min_patches,
			    "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
			    "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
			    "codebook_do_resize": self.codebook_do_resize,
			    "codebook_size": self.codebook_size,
			    "codebook_resample": self.codebook_resample,
			    "codebook_do_center_crop": self.codebook_do_center_crop,
			    "codebook_crop_size": self.codebook_crop_size,
			    "codebook_do_map_pixels": self.codebook_do_map_pixels,
			    "codebook_do_normalize": self.codebook_do_normalize,
			    "codebook_image_mean": self.codebook_image_mean,
			    "codebook_image_std": self.codebook_image_std,
			}

	def 					__UpperCAmelCase(       self   ):
			return (self.size["height"], self.size["width"])

	def 					__UpperCAmelCase(       self   ):
			return (
			    (self.input_size_patches, self.input_size_patches)
			    if not isinstance(self.input_size_patches							,			__UpperCAmelCase   )
			    else self.input_size_patches
			)







	def 					__UpperCAmelCase(       self   ):
			return (self.codebook_size["height"], self.codebook_size["width"])


@require_torch
@require_vision
class _a       (	lowerCAmelCase__   ,							unittest.TestCase ):

	'''simple docstring'''



	lowerCamelCase_   :      Dict       =	FlavaImageProcessor if is_vision_available() else None
	lowerCamelCase_   :      Optional[Any]       =	None

	def 					__UpperCAmelCase(       self   ):
			__A							:   int	  =				FlavaImageProcessingTester(self   )

	@property
	def 					__UpperCAmelCase(       self   ):
			return self.image_processor_tester.prepare_image_processor_dict()

	def 					__UpperCAmelCase(       self   ):
			__A							:   Tuple	  =				self.image_processing_class(**self.image_processor_dict   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"image_mean"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"image_std"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"do_normalize"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"do_resize"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"resample"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"crop_size"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"do_center_crop"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"do_rescale"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"rescale_factor"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"masking_generator"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_do_resize"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_size"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_resample"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_do_center_crop"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_crop_size"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_do_map_pixels"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_do_normalize"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_image_mean"   )   )
			self.assertTrue(hasattr(__UpperCAmelCase							,			"codebook_image_std"   )   )

	def 					__UpperCAmelCase(       self   ):
			__A							:   List[Any]	  =				self.image_processing_class.from_dict(self.image_processor_dict   )
			self.assertEqual(image_processor.size							,			{"height": 224, "width": 224}   )
			self.assertEqual(image_processor.crop_size							,			{"height": 224, "width": 224}   )
			self.assertEqual(image_processor.codebook_size							,			{"height": 112, "width": 112}   )
			self.assertEqual(image_processor.codebook_crop_size							,			{"height": 112, "width": 112}   )

			__A							:   Dict	  =				self.image_processing_class.from_dict(
			    self.image_processor_dict							,			size=42							,			crop_size=84							,			codebook_size=33							,			codebook_crop_size=66   )
			self.assertEqual(image_processor.size							,			{"height": 42, "width": 42}   )
			self.assertEqual(image_processor.crop_size							,			{"height": 84, "width": 84}   )
			self.assertEqual(image_processor.codebook_size							,			{"height": 33, "width": 33}   )
			self.assertEqual(image_processor.codebook_crop_size							,			{"height": 66, "width": 66}   )

	def 					__UpperCAmelCase(       self   ):
			pass

	def 					__UpperCAmelCase(       self   ):
			# Initialize image_processing
			__A							:   int	  =				self.image_processing_class(**self.image_processor_dict   )
			# create random PIL images
			__A							:   Tuple	  =				prepare_image_inputs(self.image_processor_tester							,			equal_resolution=__UpperCAmelCase   )
			for image in image_inputs:
					self.assertIsInstance(__UpperCAmelCase							,			PIL.Image.Image   )

			# Test not batched input
			__A							:   Tuple	  =				image_processing(image_inputs[0]							,			return_tensors="pt"   )

			# Test no bool masked pos
			self.assertFalse("bool_masked_pos" in encoded_images   )

			__A  ,       __A							:   Tuple	  =				self.image_processor_tester.get_expected_image_size()

			self.assertEqual(
			    encoded_images.pixel_values.shape							,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,			)

			# Test batched
			__A							:   List[Any]	  =				image_processing(__UpperCAmelCase							,			return_tensors="pt"   )
			__A  ,       __A							:   List[str]	  =				self.image_processor_tester.get_expected_image_size()

			# Test no bool masked pos
			self.assertFalse("bool_masked_pos" in encoded_images   )

			self.assertEqual(
			    encoded_images.pixel_values.shape							,			(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        expected_height,
			        expected_width,
			    )							,			)

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase   ):
			# Initialize image_processing
			__A							:   Tuple	  =				self.image_processing_class(**self.image_processor_dict   )
			# create random tensors
			__A							:   Union[str, Any]	  =				prepare_image_inputs(self.image_processor_tester							,			equal_resolution=__UpperCAmelCase							,			**__UpperCAmelCase   )
			for image in image_inputs:
					self.assertIsInstance(__UpperCAmelCase							,			__UpperCAmelCase   )

			# Test not batched input
			__A							:   Optional[int]	  =				image_processing(image_inputs[0]							,			return_tensors="pt"   )

			__A  ,       __A							:   Any	  =				self.image_processor_tester.get_expected_image_size()
			self.assertEqual(
			    encoded_images.pixel_values.shape							,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,			)

			__A							:   Optional[Any]	  =				image_processing(__UpperCAmelCase							,			return_image_mask=__UpperCAmelCase							,			return_tensors="pt"   )

			__A  ,       __A							:   int	  =				self.image_processor_tester.get_expected_image_size()
			self.assertEqual(
			    encoded_images.pixel_values.shape							,			(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        expected_height,
			        expected_width,
			    )							,			)

			__A  ,       __A							:   List[Any]	  =				self.image_processor_tester.get_expected_mask_size()
			self.assertEqual(
			    encoded_images.bool_masked_pos.shape							,			(
			        self.image_processor_tester.batch_size,
			        expected_height,
			        expected_width,
			    )							,			)

			# Test batched
			__A							:   int	  =				image_processing(__UpperCAmelCase							,			return_tensors="pt"   ).pixel_values

			__A  ,       __A							:   Union[str, Any]	  =				self.image_processor_tester.get_expected_image_size()
			self.assertEqual(
			    encoded_images.shape							,			(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        expected_height,
			        expected_width,
			    )							,			)

			# Test masking
			__A							:   int	  =				image_processing(__UpperCAmelCase							,			return_image_mask=__UpperCAmelCase							,			return_tensors="pt"   )

			__A  ,       __A							:   str	  =				self.image_processor_tester.get_expected_image_size()
			self.assertEqual(
			    encoded_images.pixel_values.shape							,			(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        expected_height,
			        expected_width,
			    )							,			)

			__A  ,       __A							:   Dict	  =				self.image_processor_tester.get_expected_mask_size()
			self.assertEqual(
			    encoded_images.bool_masked_pos.shape							,			(
			        self.image_processor_tester.batch_size,
			        expected_height,
			        expected_width,
			    )							,			)

	def 					__UpperCAmelCase(       self   ):
			self._test_call_framework(np.ndarray							,			prepare_kwargs={"numpify": True}   )

	def 					__UpperCAmelCase(       self   ):
			self._test_call_framework(torch.Tensor							,			prepare_kwargs={"torchify": True}   )

	def 					__UpperCAmelCase(       self   ):
			# Initialize image_processing
			random.seed(1_234   )
			__A							:   Dict	  =				self.image_processing_class(**self.image_processor_dict   )
			__A							:   Tuple	  =				prepare_image_inputs(self.image_processor_tester							,			equal_resolution=__UpperCAmelCase							,			torchify=__UpperCAmelCase   )

			# Test not batched input
			__A							:   Dict	  =				image_processing(image_inputs[0]							,			return_image_mask=__UpperCAmelCase							,			return_tensors="pt"   )
			self.assertEqual(encoded_images.bool_masked_pos.sum().item()							,			75   )







	def 					__UpperCAmelCase(       self   ):
			# Initialize image_processing
			__A							:   Dict	  =				self.image_processing_class(**self.image_processor_dict   )
			# create random PIL images
			__A							:   Dict	  =				prepare_image_inputs(self.image_processor_tester							,			equal_resolution=__UpperCAmelCase   )
			for image in image_inputs:
					self.assertIsInstance(__UpperCAmelCase							,			PIL.Image.Image   )

			# Test not batched input
			__A							:   List[Any]	  =				image_processing(image_inputs[0]							,			return_codebook_pixels=__UpperCAmelCase							,			return_tensors="pt"   )
			__A  ,       __A							:   Optional[int]	  =				self.image_processor_tester.get_expected_codebook_image_size()
			self.assertEqual(
			    encoded_images.codebook_pixel_values.shape							,			(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,			)

			# Test batched
			__A							:   Union[str, Any]	  =				image_processing(__UpperCAmelCase							,			return_codebook_pixels=__UpperCAmelCase							,			return_tensors="pt"   )
			__A  ,       __A							:   Any	  =				self.image_processor_tester.get_expected_codebook_image_size()
			self.assertEqual(
			    encoded_images.codebook_pixel_values.shape							,			(
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        expected_height,
			        expected_width,
			    )							,			)






import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata



def    lowerCamelCase_   (  _lowercase    )							->	str:
		__A							:   Optional[int]	  =				min(re.search("^ +" ,     _lowercase    ).end() if t.startswith(" "    ) else 0 for t in string.splitlines()    )
		return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(_lowercase    )]    )


UpperCamelCase      			=     '\\n---\nlanguage:\n- zh\n- en\ntask_ids:\n- sentiment-classification\n---\n# Begin of markdown\n\nSome cool dataset card\n'

UpperCamelCase      			=     '\\n---\n---\n# Begin of markdown\n\nSome cool dataset card\n'


UpperCamelCase      			=     '\\n# Begin of markdown\n\nSome cool dataset card\n'





class _a       (	unittest.TestCase ):

	'''simple docstring'''



	def 					__UpperCAmelCase(       self   ):
			with tempfile.TemporaryDirectory() as tmp_dir:
					__A							:   List[Any]	  =				Path(__UpperCAmelCase   ) / "README.md"
					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Tuple	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertDictEqual(__UpperCAmelCase							,			{"language": ["zh", "en"], "task_ids": ["sentiment-classification"]}   )

					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Any	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertDictEqual(__UpperCAmelCase							,			{}   )

					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Union[str, Any]	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertEqual(__UpperCAmelCase							,			{}   )







	def 					__UpperCAmelCase(       self   ):
			__A							:   List[str]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            "   )
			assert DatasetMetadata.from_yaml_string(__UpperCAmelCase   )

			__A							:   List[Any]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            "   )
			with self.assertRaises(__UpperCAmelCase   ):
					DatasetMetadata.from_yaml_string(__UpperCAmelCase   )

			__A							:   Optional[int]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the \"Researcher\") has requested permission to use the ImageNet database (the \"Database\") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            "   )
			assert DatasetMetadata.from_yaml_string(__UpperCAmelCase   )





