




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
		import torch

if is_vision_available():
		from PIL import Image

		from transformers import DeiTImageProcessor
class 		UpperCAmelCase       (    unittest.TestCase      ):







						'''simple docstring'''





						def __init__(						self							:					Any	,     __lowercase							:					Dict	,     __lowercase							:					Any=7	,     __lowercase							:					List[str]=3	,     __lowercase							:					Optional[Any]=18	,     __lowercase							:					List[str]=30	,     __lowercase							:					List[str]=4_00	,     __lowercase							:					int=True	,     __lowercase							:					str=None	,     __lowercase							:					Optional[Any]=True	,     __lowercase							:					str=None	,     __lowercase							:					Union[str, Any]=True	,     __lowercase							:					List[str]=[0.5, 0.5, 0.5]	,     __lowercase							:					List[Any]=[0.5, 0.5, 0.5]	,     ):






							"""simple docstring"""






							snake_case_		   =					size if size is not None else {"height": 20, "width": 20}
							snake_case_		   =					crop_size if crop_size is not None else {"height": 18, "width": 18}

							snake_case_		   =					parent
							snake_case_		   =					batch_size
							snake_case_		   =					num_channels
							snake_case_		   =					image_size
							snake_case_		   =					min_resolution
							snake_case_		   =					max_resolution
							snake_case_		   =					do_resize
							snake_case_		   =					size
							snake_case_		   =					do_center_crop
							snake_case_		   =					crop_size
							snake_case_		   =					do_normalize
							snake_case_		   =					image_mean
							snake_case_		   =					image_std






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							return {
							    "do_resize": self.do_resize,
							    "size": self.size,
							    "do_center_crop": self.do_center_crop,
							    "crop_size": self.crop_size,
							    "do_normalize": self.do_normalize,
							    "image_mean": self.image_mean,
							    "image_std": self.image_std,
							}







@require_torch
@require_vision
class 		UpperCAmelCase       (    UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       DeiTImageProcessor if is_vision_available() else None
						lowerCAmelCase_							=       True






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					DeiTImageProcessingTester(self						)






						@property
						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							return self.image_processor_tester.prepare_image_processor_dict()






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					self.image_processing_class(**self.image_processor_dict						)
							self.assertTrue(hasattr(__lowercase	,     "do_resize"						)						)
							self.assertTrue(hasattr(__lowercase	,     "size"						)						)
							self.assertTrue(hasattr(__lowercase	,     "do_center_crop"						)						)
							self.assertTrue(hasattr(__lowercase	,     "center_crop"						)						)
							self.assertTrue(hasattr(__lowercase	,     "do_normalize"						)						)
							self.assertTrue(hasattr(__lowercase	,     "image_mean"						)						)
							self.assertTrue(hasattr(__lowercase	,     "image_std"						)						)






						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							snake_case_		   =					self.image_processing_class.from_dict(self.image_processor_dict						)
							self.assertEqual(image_processor.size	,     {"height": 20, "width": 20}						)
							self.assertEqual(image_processor.crop_size	,     {"height": 18, "width": 18}						)

							snake_case_		   =					self.image_processing_class.from_dict(self.image_processor_dict	,     size=42	,     crop_size=84						)
							self.assertEqual(image_processor.size	,     {"height": 42, "width": 42}						)
							self.assertEqual(image_processor.crop_size	,     {"height": 84, "width": 84}						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							pass






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					self.image_processing_class(**self.image_processor_dict						)
							# create random PIL images
							snake_case_		   =					prepare_image_inputs(self.image_processor_tester	,     equal_resolution=__lowercase						)
							for image in image_inputs:
								self.assertIsInstance(__lowercase	,     Image.Image						)

							# Test not batched input
							snake_case_		   =					image_processing(image_inputs[0]	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )

							# Test batched
							snake_case_		   =					image_processing(__lowercase	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )






						def 	snake_case__						(						self							:					Any						):






							"""simple docstring"""






							snake_case_		   =					self.image_processing_class(**self.image_processor_dict						)
							# create random numpy tensors
							snake_case_		   =					prepare_image_inputs(self.image_processor_tester	,     equal_resolution=__lowercase	,     numpify=__lowercase						)
							for image in image_inputs:
								self.assertIsInstance(__lowercase	,     np.ndarray						)

							# Test not batched input
							snake_case_		   =					image_processing(image_inputs[0]	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )

							# Test batched
							snake_case_		   =					image_processing(__lowercase	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					self.image_processing_class(**self.image_processor_dict						)
							# create random PyTorch tensors
							snake_case_		   =					prepare_image_inputs(self.image_processor_tester	,     equal_resolution=__lowercase	,     torchify=__lowercase						)
							for image in image_inputs:
								self.assertIsInstance(__lowercase	,     torch.Tensor						)

							# Test not batched input
							snake_case_		   =					image_processing(image_inputs[0]	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )

							# Test batched
							snake_case_		   =					image_processing(__lowercase	,     return_tensors="pt"						).pixel_values
							self.assertEqual(
							    encoded_images.shape	,     (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["height"],
							        self.image_processor_tester.crop_size["width"],
							    )	,     )






import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowercase__    :					str					=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 		UpperCAmelCase       (    datasets.BuilderConfig      ):







						'''simple docstring'''





						lowerCAmelCase_							=       None







class 		UpperCAmelCase       (    datasets.ArrowBasedBuilder      ):







						'''simple docstring'''





						lowerCAmelCase_							=       ArrowConfig






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							return datasets.DatasetInfo(features=self.config.features						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Union[str, Any]						):






							"""simple docstring"""






							if not self.config.data_files:
								raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"						)
							snake_case_		   =					dl_manager.download_and_extract(self.config.data_files						)
							if isinstance(__lowercase	,     (str, list, tuple)						):
								snake_case_		   =					data_files
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,     gen_kwargs={"files": files}						)]
							snake_case_		   =					[]
							for split_name, files in data_files.items():
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								# Infer features is they are stoed in the arrow schema
								if self.info.features is None:
									for file in itertools.chain.from_iterable(__lowercase						):
										with open(__lowercase	,     "rb"						) as f:
											snake_case_		   =					datasets.Features.from_arrow_schema(pa.ipc.open_stream(__lowercase						).schema						)
										break
								splits.append(datasets.SplitGenerator(name=__lowercase	,     gen_kwargs={"files": files}						)						)
							return splits






						def 	snake_case__						(						self							:					str	,     __lowercase							:					pa.Table						):






							"""simple docstring"""






							if self.info.features is not None:
								# more expensive cast to support nested features with keys in a different order
								# allows str <-> int/float or str to Audio for example
								snake_case_		   =					table_cast(__lowercase	,     self.info.features.arrow_schema						)
							return pa_table






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[Any]						):






							"""simple docstring"""






							for file_idx, file in enumerate(itertools.chain.from_iterable(__lowercase						)						):
								with open(__lowercase	,     "rb"						) as f:
									try:
										for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__lowercase						)						):
											snake_case_		   =					pa.Table.from_batches([record_batch]						)
											# Uncomment for debugging (will print the Arrow table size and elements)
											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield f"{file_idx}_{batch_idx}", self._cast_table(__lowercase						)
									except ValueError as e:
										logger.error(f"Failed to read file '{file}' with error {type(__lowercase						)}: {e}"						)
										raise
