

import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase       :				Dict         =					datasets.utils.logging.get_logger(__name__)
@dataclass
class 	_A						(					datasets.BuilderConfig):
  SCREAMING_SNAKE_CASE      : Optional[datasets.Features]    =					None
  SCREAMING_SNAKE_CASE      : str    =					"utf-8"
  SCREAMING_SNAKE_CASE      : Optional[str]    =					None
  SCREAMING_SNAKE_CASE      : int    =					10 << 20  # 10MB
  SCREAMING_SNAKE_CASE      : bool    =					False
  SCREAMING_SNAKE_CASE      : str    =					"line"

class 	_A						(					datasets.ArrowBasedBuilder):
  SCREAMING_SNAKE_CASE      : Tuple    =					TextConfig

  def 	UpperCAmelCase      (			self   ):





        """simple docstring"""







        return datasets.DatasetInfo(features=self.config.features   )

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        if not self.config.data_files:
              raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"   )
        SCREAMING_SNAKE_CASE_			:					Tuple          =       dl_manager.download_and_extract(self.config.data_files   )
        if isinstance(_SCREAMING_SNAKE_CASE	,   (str, list, tuple)   ):
              SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       data_files
              if isinstance(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ):
                    SCREAMING_SNAKE_CASE_			:					Tuple          =       [files]
              SCREAMING_SNAKE_CASE_			:					Any          =       [dl_manager.iter_files(_SCREAMING_SNAKE_CASE   ) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,   gen_kwargs={'files': files}   )]
        SCREAMING_SNAKE_CASE_			:					Dict          =       []
        for split_name, files in data_files.items():
              if isinstance(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ):
                    SCREAMING_SNAKE_CASE_			:					Dict          =       [files]
              SCREAMING_SNAKE_CASE_			:					Dict          =       [dl_manager.iter_files(_SCREAMING_SNAKE_CASE   ) for file in files]
              splits.append(datasets.SplitGenerator(name=_SCREAMING_SNAKE_CASE	,   gen_kwargs={'files': files}   )   )
        return splits

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        if self.config.features is not None:
              SCREAMING_SNAKE_CASE_			:					Optional[int]          =       self.config.features.arrow_schema
              if all(not require_storage_cast(_SCREAMING_SNAKE_CASE   ) for feature in self.config.features.values()   ):
                    # cheaper cast
                    SCREAMING_SNAKE_CASE_			:					List[Any]          =       pa_table.cast(_SCREAMING_SNAKE_CASE   )
              else:
                    # more expensive cast; allows str <-> int/float or str to Audio for example
                    SCREAMING_SNAKE_CASE_			:					Optional[int]          =       table_cast(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )
              return pa_table
        else:
              return pa_table.cast(pa.schema({'text': pa.string()}   )   )



  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					List[Any]          =       list(self.config.features   ) if self.config.features is not None else ['text']
        for file_idx, file in enumerate(itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE   )   ):
              # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
              with open(_SCREAMING_SNAKE_CASE	,   encoding=self.config.encoding	,   errors=self.config.errors   ) as f:
                    if self.config.sample_by == "line":
                          SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       0
                          while True:
                                SCREAMING_SNAKE_CASE_			:					List[Any]          =       f.read(self.config.chunksize   )
                                if not batch:
                                      break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                SCREAMING_SNAKE_CASE_			:					List[str]          =       StringIO(_SCREAMING_SNAKE_CASE   ).readlines()
                                if not self.config.keep_linebreaks:
                                      SCREAMING_SNAKE_CASE_			:					Tuple          =       [line.rstrip('\n'   ) for line in batch]
                                SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       pa.Table.from_arrays([pa.array(_SCREAMING_SNAKE_CASE   )]	,   names=_SCREAMING_SNAKE_CASE   )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(_SCREAMING_SNAKE_CASE   )
                                batch_idx += 1
                    elif self.config.sample_by == "paragraph":
                          SCREAMING_SNAKE_CASE_			:					List[str]          =       0
                          SCREAMING_SNAKE_CASE_			:					Tuple          =       ''
                          while True:
                                SCREAMING_SNAKE_CASE_			:					Any          =       f.read(self.config.chunksize   )
                                if not new_batch:
                                      break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                SCREAMING_SNAKE_CASE_			:					Tuple          =       batch.split('\n\n'   )
                                SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]   )]	,   names=_SCREAMING_SNAKE_CASE   )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(_SCREAMING_SNAKE_CASE   )
                                batch_idx += 1
                                SCREAMING_SNAKE_CASE_			:					Optional[int]          =       batch[-1]
                          if batch:
                                SCREAMING_SNAKE_CASE_			:					str          =       pa.Table.from_arrays([pa.array([batch]   )]	,   names=_SCREAMING_SNAKE_CASE   )
                                yield (file_idx, batch_idx), self._cast_table(_SCREAMING_SNAKE_CASE   )
                    elif self.config.sample_by == "document":
                          SCREAMING_SNAKE_CASE_			:					List[Any]          =       f.read()
                          SCREAMING_SNAKE_CASE_			:					int          =       pa.Table.from_arrays([pa.array([text]   )]	,   names=_SCREAMING_SNAKE_CASE   )
                          yield file_idx, self._cast_table(_SCREAMING_SNAKE_CASE   )









from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
    import PIL


lowerCAmelCase       :				Dict         =					logging.get_logger(__name__)
class 	_A						(					__magic_name__):
  SCREAMING_SNAKE_CASE      : Any    =					['''pixel_values''']

  def __init__(			self	,   _SCREAMING_SNAKE_CASE = True	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = PIL.Image.NEAREST	,   _SCREAMING_SNAKE_CASE = False	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = 1 / 255	,   _SCREAMING_SNAKE_CASE = False	,   _SCREAMING_SNAKE_CASE = True	,   _SCREAMING_SNAKE_CASE = True	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = True	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        super().__init__(**_SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					str          =       size if size is not None else {'height': 346, 'width': 346}
        SCREAMING_SNAKE_CASE_			:					str          =       get_size_dict(_SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					int          =       crop_size if crop_size is not None else {'height': 289, 'width': 289}
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       get_size_dict(_SCREAMING_SNAKE_CASE	,   param_name='crop_size'   )

        SCREAMING_SNAKE_CASE_			:					List[str]          =       do_resize
        SCREAMING_SNAKE_CASE_			:					int          =       size
        SCREAMING_SNAKE_CASE_			:					Tuple          =       resample
        SCREAMING_SNAKE_CASE_			:					int          =       do_center_crop
        SCREAMING_SNAKE_CASE_			:					List[str]          =       crop_size
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       do_rescale
        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       rescale_factor
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       rescale_offset
        SCREAMING_SNAKE_CASE_			:					Dict          =       do_normalize
        SCREAMING_SNAKE_CASE_			:					str          =       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
        SCREAMING_SNAKE_CASE_			:					List[str]          =       image_std if image_std is not None else IMAGENET_STANDARD_STD
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       include_top

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = PIL.Image.NEAREST	,   _SCREAMING_SNAKE_CASE = None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					str          =       get_size_dict(_SCREAMING_SNAKE_CASE   )
        if "height" not in size or "width" not in size:
              raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"   )
        return resize(
            _SCREAMING_SNAKE_CASE	,   size=(size['height'], size['width'])	,   resample=_SCREAMING_SNAKE_CASE	,   data_format=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE   )

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Tuple          =       get_size_dict(_SCREAMING_SNAKE_CASE   )
        if "height" not in size or "width" not in size:
              raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"   )
        return center_crop(_SCREAMING_SNAKE_CASE	,   size=(size['height'], size['width'])	,   data_format=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE   )

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = True	,   _SCREAMING_SNAKE_CASE = None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        if offset:
              SCREAMING_SNAKE_CASE_			:					Dict          =       (image - 127.5) * scale
              if data_format is not None:
                    SCREAMING_SNAKE_CASE_			:					Tuple          =       to_channel_dimension_format(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   )
              SCREAMING_SNAKE_CASE_			:					int          =       rescaled_image.astype(np.floataa   )
        else:
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       rescale(_SCREAMING_SNAKE_CASE	,   scale=_SCREAMING_SNAKE_CASE	,   data_format=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE   )
        return rescaled_image

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        return normalize(_SCREAMING_SNAKE_CASE	,   mean=_SCREAMING_SNAKE_CASE	,   std=_SCREAMING_SNAKE_CASE	,   data_format=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE   )



  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE=None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = None	,   _SCREAMING_SNAKE_CASE = ChannelDimension.FIRST	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					List[Any]          =       do_resize if do_resize is not None else self.do_resize
        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       resample if resample is not None else self.resample
        SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       do_center_crop if do_center_crop is not None else self.do_center_crop
        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       do_rescale if do_rescale is not None else self.do_rescale
        SCREAMING_SNAKE_CASE_			:					int          =       rescale_factor if rescale_factor is not None else self.rescale_factor
        SCREAMING_SNAKE_CASE_			:					Tuple          =       rescale_offset if rescale_offset is not None else self.rescale_offset
        SCREAMING_SNAKE_CASE_			:					Dict          =       do_normalize if do_normalize is not None else self.do_normalize
        SCREAMING_SNAKE_CASE_			:					str          =       image_mean if image_mean is not None else self.image_mean
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       image_std if image_std is not None else self.image_std
        SCREAMING_SNAKE_CASE_			:					List[Any]          =       include_top if include_top is not None else self.include_top

        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       size if size is not None else self.size
        SCREAMING_SNAKE_CASE_			:					Tuple          =       get_size_dict(_SCREAMING_SNAKE_CASE   )
        SCREAMING_SNAKE_CASE_			:					Any          =       crop_size if crop_size is not None else self.crop_size
        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       get_size_dict(_SCREAMING_SNAKE_CASE	,   param_name='crop_size'   )

        SCREAMING_SNAKE_CASE_			:					List[str]          =       make_list_of_images(_SCREAMING_SNAKE_CASE   )

        if not valid_images(_SCREAMING_SNAKE_CASE   ):
              raise ValueError(
                  'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
                  'torch.Tensor, tf.Tensor or jax.ndarray.'   )

        if do_resize and size is None or resample is None:
              raise ValueError('Size and resample must be specified if do_resize is True.'   )

        if do_center_crop and crop_size is None:
              raise ValueError('Crop size must be specified if do_center_crop is True.'   )

        if do_rescale and rescale_factor is None:
              raise ValueError('Rescale factor must be specified if do_rescale is True.'   )

        if do_normalize and (image_mean is None or image_std is None):
              raise ValueError('Image mean and std must be specified if do_normalize is True.'   )

        # All transformations expect numpy arrays.
        SCREAMING_SNAKE_CASE_			:					int          =       [to_numpy_array(_SCREAMING_SNAKE_CASE   ) for image in images]

        if do_resize:
              SCREAMING_SNAKE_CASE_			:					Dict          =       [self.resize(image=_SCREAMING_SNAKE_CASE	,   size=_SCREAMING_SNAKE_CASE	,   resample=_SCREAMING_SNAKE_CASE   ) for image in images]

        if do_center_crop:
              SCREAMING_SNAKE_CASE_			:					str          =       [self.center_crop(image=_SCREAMING_SNAKE_CASE	,   size=_SCREAMING_SNAKE_CASE   ) for image in images]

        if do_rescale:
              SCREAMING_SNAKE_CASE_			:					Tuple          =       [self.rescale(image=_SCREAMING_SNAKE_CASE	,   scale=_SCREAMING_SNAKE_CASE	,   offset=_SCREAMING_SNAKE_CASE   ) for image in images]

        if do_normalize:
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       [self.normalize(image=_SCREAMING_SNAKE_CASE	,   mean=_SCREAMING_SNAKE_CASE	,   std=_SCREAMING_SNAKE_CASE   ) for image in images]

        if include_top:
              SCREAMING_SNAKE_CASE_			:					Dict          =       [self.normalize(image=_SCREAMING_SNAKE_CASE	,   mean=[0, 0, 0]	,   std=_SCREAMING_SNAKE_CASE   ) for image in images]

        SCREAMING_SNAKE_CASE_			:					Dict          =       [to_channel_dimension_format(_SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE   ) for image in images]

        SCREAMING_SNAKE_CASE_			:					Any          =       {'pixel_values': images}
        return BatchFeature(data=_SCREAMING_SNAKE_CASE	,   tensor_type=_SCREAMING_SNAKE_CASE   )






