






"""simple docstring"""



import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_UpperCamelCase		:  Dict						=		datasets.utils.logging.get_logger(__name__)

@dataclass
class 	a					(       datasets.BuilderConfig     ):
    UpperCAmelCase_			:					Optional[datasets.Features]     					=None


class 	a					(       datasets.ArrowBasedBuilder     ):
    UpperCAmelCase_			:					Union[str, Any]     					=ArrowConfig







    def       UpperCamelCase_				(      self	):
      return datasets.DatasetInfo(features=self.config.features	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):


      if not self.config.data_files:
        raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'	)
      lowercase							   =	dl_manager.download_and_extract(self.config.data_files	)
      if isinstance(_UpperCAmelCase							,		(str, list, tuple)	):
        lowercase							   =	data_files
        if isinstance(_UpperCAmelCase							,		_UpperCAmelCase	):
          lowercase							   =	[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        lowercase							   =	[dl_manager.iter_files(_UpperCAmelCase	) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,		gen_kwargs={'files': files}	)]
      lowercase							   =	[]
      for split_name, files in data_files.items():
        if isinstance(_UpperCAmelCase							,		_UpperCAmelCase	):
          lowercase							   =	[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        lowercase							   =	[dl_manager.iter_files(_UpperCAmelCase	) for file in files]
        # Infer features is they are stoed in the arrow schema
        if self.info.features is None:
          for file in itertools.chain.from_iterable(_UpperCAmelCase	):
            with open(_UpperCAmelCase							,		'rb'	) as f:
              lowercase							   =	datasets.Features.from_arrow_schema(pa.ipc.open_stream(_UpperCAmelCase	).schema	)
            break
        splits.append(datasets.SplitGenerator(name=_UpperCAmelCase							,		gen_kwargs={'files': files}	)	)
      return splits







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):


      if self.info.features is not None:
        # more expensive cast to support nested features with keys in a different order
        # allows str <-> int/float or str to Audio for example
        lowercase							   =	table_cast(_UpperCAmelCase							,		self.info.features.arrow_schema	)
      return pa_table







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):


      for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase	)	):
        with open(_UpperCAmelCase							,		'rb'	) as f:
          try:
            for batch_idx, record_batch in enumerate(pa.ipc.open_stream(_UpperCAmelCase	)	):
              lowercase							   =	pa.Table.from_batches([record_batch]	)
              # Uncomment for debugging (will print the Arrow table size and elements)
              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
              yield F'{file_idx}_{batch_idx}', self._cast_table(_UpperCAmelCase	)
          except ValueError as e:
            logger.error(F'Failed to read file \'{file}\' with error {type(_UpperCAmelCase	)}: {e}'	)
            raise










"""simple docstring"""



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import ViTImageProcessor

class 	a					(       unittest.TestCase     ):







    def __init__(      self							,		_lowerCamelCase							,		_lowerCamelCase=7							,		_lowerCamelCase=3							,		_lowerCamelCase=1_8							,		_lowerCamelCase=3_0							,		_lowerCamelCase=4_0_0							,		_lowerCamelCase=True							,		_lowerCamelCase=None							,		_lowerCamelCase=True							,		_lowerCamelCase=[0.5, 0.5, 0.5]							,		_lowerCamelCase=[0.5, 0.5, 0.5]							,		):
      lowercase							   =	size if size is not None else {'height': 1_8, 'width': 1_8}
      lowercase							   =	parent
      lowercase							   =	batch_size
      lowercase							   =	num_channels
      lowercase							   =	image_size
      lowercase							   =	min_resolution
      lowercase							   =	max_resolution
      lowercase							   =	do_resize
      lowercase							   =	size
      lowercase							   =	do_normalize
      lowercase							   =	image_mean
      lowercase							   =	image_std







    def       UpperCamelCase_				(      self	):
      return {
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_normalize": self.do_normalize,
          "do_resize": self.do_resize,
          "size": self.size,
      }


@require_torch
@require_vision
class 	a					(       a_,    unittest.TestCase     ):
    UpperCAmelCase_			:					int     					=ViTImageProcessor if is_vision_available() else None







    def       UpperCamelCase_				(      self	):
      lowercase							   =	ViTImageProcessingTester(self	)







    @property
    def       UpperCamelCase_				(      self	):
      return self.image_processor_tester.prepare_image_processor_dict()







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.image_processing_class(**self.image_processor_dict	)
      self.assertTrue(hasattr(_lowerCamelCase							,		'image_mean'	)	)
      self.assertTrue(hasattr(_lowerCamelCase							,		'image_std'	)	)
      self.assertTrue(hasattr(_lowerCamelCase							,		'do_normalize'	)	)
      self.assertTrue(hasattr(_lowerCamelCase							,		'do_resize'	)	)
      self.assertTrue(hasattr(_lowerCamelCase							,		'size'	)	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.image_processing_class.from_dict(self.image_processor_dict	)
      self.assertEqual(image_processor.size							,		{'height': 1_8, 'width': 1_8}	)

      lowercase							   =	self.image_processing_class.from_dict(self.image_processor_dict							,		size=4_2	)
      self.assertEqual(image_processor.size							,		{'height': 4_2, 'width': 4_2}	)







    def       UpperCamelCase_				(      self	):
      pass







    def       UpperCamelCase_				(      self	):
      # Initialize image_processing
      lowercase							   =	self.image_processing_class(**self.image_processor_dict	)
      # create random PIL images
      lowercase							   =	prepare_image_inputs(self.image_processor_tester							,		equal_resolution=_lowerCamelCase	)
      for image in image_inputs:
        self.assertIsInstance(_lowerCamelCase							,		Image.Image	)

      # Test not batched input
      lowercase							   =	image_processing(image_inputs[0]							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)

      # Test batched
      lowercase							   =	image_processing(_lowerCamelCase							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)







    def       UpperCamelCase_				(      self	):
      # Initialize image_processing
      lowercase							   =	self.image_processing_class(**self.image_processor_dict	)
      # create random numpy tensors
      lowercase							   =	prepare_image_inputs(self.image_processor_tester							,		equal_resolution=_lowerCamelCase							,		numpify=_lowerCamelCase	)
      for image in image_inputs:
        self.assertIsInstance(_lowerCamelCase							,		np.ndarray	)

      # Test not batched input
      lowercase							   =	image_processing(image_inputs[0]							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)

      # Test batched
      lowercase							   =	image_processing(_lowerCamelCase							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)







    def       UpperCamelCase_				(      self	):
      # Initialize image_processing
      lowercase							   =	self.image_processing_class(**self.image_processor_dict	)
      # create random PyTorch tensors
      lowercase							   =	prepare_image_inputs(self.image_processor_tester							,		equal_resolution=_lowerCamelCase							,		torchify=_lowerCamelCase	)
      for image in image_inputs:
        self.assertIsInstance(_lowerCamelCase							,		torch.Tensor	)

      # Test not batched input
      lowercase							   =	image_processing(image_inputs[0]							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)

      # Test batched
      lowercase							   =	image_processing(_lowerCamelCase							,		return_tensors='pt'	).pixel_values
      self.assertEqual(
          encoded_images.shape							,		(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size['height'],
              self.image_processor_tester.size['width'],
          )							,		)


