

import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCamelCase									= datasets.utils.logging.get_logger(__name__)







@dataclass
class 						__magic_name__      (							datasets.BuilderConfig							):

   '''simple docstring'''




   lowerCamelCase__			:     Optional[datasets.Features]    =       None







class 						__magic_name__      (							datasets.ArrowBasedBuilder							):

   '''simple docstring'''




   lowerCamelCase__			:     Tuple    =       ArrowConfig



   def 			_UpperCAmelCase			(    self	)							->   Optional[Any]:






     """simple docstring"""






     return datasets.DatasetInfo(features=self.config.features	)



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   int:






     """simple docstring"""






     if not self.config.data_files:
       raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
     a__  =dl_manager.download_and_extract(self.config.data_files	)
     if isinstance(lowercase_,   (str, list, tuple)	):
       a__  =data_files
       if isinstance(lowercase_,   lowercase_	):
         a__  =[files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       a__  =[dl_manager.iter_files(lowercase_	) for file in files]
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN,   gen_kwargs={'''files''': files}	)]
     a__  =[]
     for split_name, files in data_files.items():
       if isinstance(lowercase_,   lowercase_	):
         a__  =[files]
       # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
       a__  =[dl_manager.iter_files(lowercase_	) for file in files]
       # Infer features is they are stoed in the arrow schema
       if self.info.features is None:
         for file in itertools.chain.from_iterable(lowercase_	):
           with open(lowercase_,   '''rb'''	) as f:
             a__  =datasets.Features.from_arrow_schema(pa.ipc.open_stream(lowercase_	).schema	)
           break
       splits.append(datasets.SplitGenerator(name=lowercase_,   gen_kwargs={'''files''': files}	)	)
     return splits



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   pa.Table:






     """simple docstring"""






     if self.info.features is not None:
       # more expensive cast to support nested features with keys in a different order
       # allows str <-> int/float or str to Audio for example
       a__  =table_cast(lowercase_,   self.info.features.arrow_schema	)
     return pa_table



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Optional[int]:






     """simple docstring"""






     for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase_	)	):
       with open(lowercase_,   '''rb'''	) as f:
         try:
           for batch_idx, record_batch in enumerate(pa.ipc.open_stream(lowercase_	)	):
             a__  =pa.Table.from_batches([record_batch]	)
             # Uncomment for debugging (will print the Arrow table size and elements)
             # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
             # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
             yield F"""{file_idx}_{batch_idx}""", self._cast_table(lowercase_	)
         except ValueError as e:
           logger.error(F"""Failed to read file '{file}' with error {type(lowercase_	)}: {e}"""	)
           raise



# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
lowerCamelCase									= abspath(join(dirname(dirname(dirname(__file__))), '''src'''))
sys.path.insert(1, git_repo_path)


# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='''ignore''', category=FutureWarning)




def       UpperCAmelCase__    (      _A				:			List[Any]							):





  '''simple docstring'''







  from diffusers.utils.testing_utils import pytest_addoption_shared

  pytest_addoption_shared(_A							)




def       UpperCAmelCase__    (      _A				:			int							):





  '''simple docstring'''







  from diffusers.utils.testing_utils import pytest_terminal_summary_main

  a__  =terminalreporter.config.getoption('''--make-reports'''							)
  if make_reports:
    pytest_terminal_summary_main(_A     ,				id=_A							)
