


import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_snake_case					=					datasets.utils.logging.get_logger(__name__)
@dataclass
class      UpperCAmelCase_ ( datasets.BuilderConfig   ):

   '''simple docstring'''




   __A			:				Optional[datasets.Features]							=		None



class      UpperCAmelCase_ ( datasets.ArrowBasedBuilder   ):

   '''simple docstring'''




   __A			:				str							=		ArrowConfig






   def _snake_case   (		self			):






     """simple docstring"""





     return datasets.DatasetInfo(features=self.config.features			)






   def _snake_case   (		self						,    __A			):






     """simple docstring"""





     if not self.config.data_files:
       raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
     lowerCamelCase			: List[Any]   =    dl_manager.download_and_extract(self.config.data_files			)
     if isinstance(__A						,    (str, list, tuple)			):
       lowerCamelCase			: List[Any]   =    data_files
       if isinstance(__A						,    __A			):
         lowerCamelCase			: Any   =    [files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       lowerCamelCase			: List[str]   =    [dl_manager.iter_files(__A			) for file in files]
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,    gen_kwargs={"files": files}			)]
     lowerCamelCase			: Union[str, Any]   =    []
     for split_name, files in data_files.items():
       if isinstance(__A						,    __A			):
         lowerCamelCase			: Optional[Any]   =    [files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       lowerCamelCase			: Dict   =    [dl_manager.iter_files(__A			) for file in files]
       # Infer features is they are stoed in the arrow schema
       if self.info.features is None:
         for file in itertools.chain.from_iterable(__A			):
           with open(__A						,    "rb"			) as f:
             lowerCamelCase			: List[Any]   =    datasets.Features.from_arrow_schema(pa.ipc.open_stream(__A			).schema			)
           break
       splits.append(datasets.SplitGenerator(name=__A						,    gen_kwargs={"files": files}			)			)
     return splits






   def _snake_case   (		self						,    __A			):






     """simple docstring"""





     if self.info.features is not None:
       # more expensive cast to support nested features with keys in a different order
       # allows str <-> int/float or str to Audio for example
       lowerCamelCase			: int   =    table_cast(__A						,    self.info.features.arrow_schema			)
     return pa_table






   def _snake_case   (		self						,    __A			):






     """simple docstring"""





     for file_idx, file in enumerate(itertools.chain.from_iterable(__A			)			):
       with open(__A						,    "rb"			) as f:
         try:
           for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__A			)			):
             lowerCamelCase			: Union[str, Any]   =    pa.Table.from_batches([record_batch]			)
             # Uncomment for debugging (will print the Arrow table size and elements)
             # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
             # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
             yield F"""{file_idx}_{batch_idx}""", self._cast_table(__A			)
         except ValueError as e:
           logger.error(F"""Failed to read file '{file}' with error {type(__A			)}: {e}"""			)
           raise







import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
   from PIL import Image
class      UpperCAmelCase_ ( unittest.TestCase		,			UpperCamelCase   ):

   '''simple docstring'''




   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: List[Any]   =    load_tool("image-segmentation"			)
     self.tool.setup()
     lowerCamelCase			: Optional[Any]   =    load_tool("image-segmentation"						,    remote=__A			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: List[str]   =    Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"			)			) / "000000039769.png"			)
     lowerCamelCase			: Dict   =    self.tool(__A						,    "cat"			)
     self.assertTrue(isinstance(__A						,    Image.Image			)			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: int   =    Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"			)			) / "000000039769.png"			)
     lowerCamelCase			: int   =    self.remote_tool(__A						,    "cat"			)
     self.assertTrue(isinstance(__A						,    Image.Image			)			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Optional[int]   =    Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"			)			) / "000000039769.png"			)
     lowerCamelCase			: Optional[Any]   =    self.tool(image=__A						,    label="cat"			)
     self.assertTrue(isinstance(__A						,    Image.Image			)			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: str   =    Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"			)			) / "000000039769.png"			)
     lowerCamelCase			: Dict   =    self.remote_tool(image=__A						,    label="cat"			)
     self.assertTrue(isinstance(__A						,    Image.Image			)			)



