
import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


__lowerCAmelCase   :     str      =						datasets.utils.logging.get_logger(__name__)


@dataclass
class   __lowerCAmelCase		(		datasets.BuilderConfig ):



      """simple docstring"""







      A__		:     Optional[datasets.Features]       				=      None






class   __lowerCAmelCase		(		datasets.ArrowBasedBuilder ):



      """simple docstring"""







      A__		:     Union[str, Any]       				=      ArrowConfig




      def 						snake_case_     ( self						:    int			):
          return datasets.DatasetInfo(features=self.config.features			)




      def 						snake_case_     ( self						:    Optional[int]			,    _snake_case						:    Union[str, Any]			):


          if not self.config.data_files:
              raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'			)
          __lowercase						:     Dict		      =    dl_manager.download_and_extract(self.config.data_files			)
          if isinstance(_UpperCAmelCase			,    (str, list, tuple)			):
              __lowercase						:     Any		      =    data_files
              if isinstance(_UpperCAmelCase			,    _UpperCAmelCase			):
                  __lowercase						:     Any		      =    [files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              __lowercase						:     Tuple		      =    [dl_manager.iter_files(_UpperCAmelCase			) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,    gen_kwargs={'''files''': files}			)]
          __lowercase						:     Any		      =    []
          for split_name, files in data_files.items():
              if isinstance(_UpperCAmelCase			,    _UpperCAmelCase			):
                  __lowercase						:     Tuple		      =    [files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              __lowercase						:     Optional[int]		      =    [dl_manager.iter_files(_UpperCAmelCase			) for file in files]
              # Infer features is they are stoed in the arrow schema
              if self.info.features is None:
                  for file in itertools.chain.from_iterable(_UpperCAmelCase			):
                      with open(_UpperCAmelCase			,    '''rb'''			) as f:
                          __lowercase						:     Optional[int]		      =    datasets.Features.from_arrow_schema(pa.ipc.open_stream(_UpperCAmelCase			).schema			)
                      break
              splits.append(datasets.SplitGenerator(name=_UpperCAmelCase			,    gen_kwargs={'''files''': files}			)			)
          return splits




      def 						snake_case_     ( self						:    Any			,    _snake_case						:    pa.Table			):


          if self.info.features is not None:
              # more expensive cast to support nested features with keys in a different order
              # allows str <-> int/float or str to Audio for example
              __lowercase						:     int		      =    table_cast(_UpperCAmelCase			,    self.info.features.arrow_schema			)
          return pa_table




      def 						snake_case_     ( self						:    Any			,    _snake_case						:    Any			):


          for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase			)			):
              with open(_UpperCAmelCase			,    '''rb'''			) as f:
                  try:
                      for batch_idx, record_batch in enumerate(pa.ipc.open_stream(_UpperCAmelCase			)			):
                          __lowercase						:     List[Any]		      =    pa.Table.from_batches([record_batch]			)
                          # Uncomment for debugging (will print the Arrow table size and elements)
                          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                          yield F'{file_idx}_{batch_idx}', self._cast_table(_UpperCAmelCase			)
                  except ValueError as e:
                      logger.error(F'Failed to read file \'{file}\' with error {type(_UpperCAmelCase			)}: {e}'			)
                      raise




__lowerCAmelCase   :     List[Any]      =						"\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

__lowerCAmelCase   :     Optional[Any]      =						[{"type": "code", "content": INSTALL_CONTENT}]
__lowerCAmelCase   :     List[str]      =						{
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}


