





'''simple docstring'''
# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
    import tensorflow as tf

class 					snake_case					(   TensorFormatter[Mapping, "tf.Tensor", Mapping]	):




 """simple docstring"""



 def __init__(	self	:							Tuple							,     __A	:							Optional[int]=None							,     **__A	:							int     ):
        super().__init__(features=__A     )
        __UpperCamelCase	=  tf_tensor_kwargs
        import tensorflow as tf  # noqa: import tf at initialization






 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							Tuple     ):
        import tensorflow as tf

        if isinstance(__A							,     __A     ) and column:
               if all(
                   isinstance(__A							,     tf.Tensor     ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column     ):
                      return tf.stack(__A     )
               elif all(
                   isinstance(__A							,     (tf.Tensor, tf.RaggedTensor)     ) and x.ndim == 1 and x.dtype == column[0].dtype
                   for x in column     ):
                      # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
                      return tf.ragged.stack(__A     )

        return column






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							Union[str, Any]     ):
        import tensorflow as tf

        if value is None:
               return value

        __UpperCamelCase	=  {}

        if isinstance(__A							,     (np.number, np.ndarray)     ) and np.issubdtype(value.dtype							,     np.integer     ):
               __UpperCamelCase	=  {'dtype': tf.intaa}
        elif isinstance(__A							,     (np.number, np.ndarray)     ) and np.issubdtype(value.dtype							,     np.floating     ):
               __UpperCamelCase	=  {'dtype': tf.floataa}
        elif config.PIL_AVAILABLE and "PIL" in sys.modules:
               import PIL.Image

               if isinstance(__A							,     PIL.Image.Image     ):
                      __UpperCamelCase	=  np.asarray(__A     )

        return tf.convert_to_tensor(__A							,     **{**default_dtype, **self.tf_tensor_kwargs}     )






 def 	_lowerCamelCase (	self	:							Dict							,     __A	:							Dict     ):
        import tensorflow as tf

        # support for torch, tf, jax etc.
        if config.TORCH_AVAILABLE and "torch" in sys.modules:
               import torch

               if isinstance(__A							,     torch.Tensor     ):
                      return self._tensorize(data_struct.detach().cpu().numpy()[()]     )
        if hasattr(__A							,     '__array__'     ) and not isinstance(__A							,     tf.Tensor     ):
               __UpperCamelCase	=  data_struct.__array__()
        # support for nested types like struct of list of struct
        if isinstance(__A							,     np.ndarray     ):
               if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
                      return self._consolidate([self.recursive_tensorize(__A     ) for substruct in data_struct]     )
        elif isinstance(__A							,     (list, tuple)     ):
               return self._consolidate([self.recursive_tensorize(__A     ) for substruct in data_struct]     )
        return self._tensorize(__A     )






 def 	_lowerCamelCase (	self	:							Dict							,     __A	:							dict     ):
        return map_nested(self._recursive_tensorize							,     __A							,     map_list=__A     )






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							pa.Table     ):
        __UpperCamelCase	=  self.numpy_arrow_extractor().extract_row(__A     )
        __UpperCamelCase	=  self.python_features_decoder.decode_row(__A     )
        return self.recursive_tensorize(__A     )






 def 	_lowerCamelCase (	self	:							str							,     __A	:							pa.Table     ):
        __UpperCamelCase	=  self.numpy_arrow_extractor().extract_column(__A     )
        __UpperCamelCase	=  self.python_features_decoder.decode_column(__A							,     pa_table.column_names[0]     )
        __UpperCamelCase	=  self.recursive_tensorize(__A     )
        __UpperCamelCase	=  self._consolidate(__A     )
        return column






 def 	_lowerCamelCase (	self	:							Union[str, Any]							,     __A	:							pa.Table     ):
        __UpperCamelCase	=  self.numpy_arrow_extractor().extract_batch(__A     )
        __UpperCamelCase	=  self.python_features_decoder.decode_batch(__A     )
        __UpperCamelCase	=  self.recursive_tensorize(__A     )
        for column_name in batch:
               __UpperCamelCase	=  self._consolidate(batch[column_name]     )
        return batch







'''simple docstring'''
import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


a__							:							Optional[Any]			=datasets.utils.logging.get_logger(__name__)

@dataclass
class 					snake_case					(   datasets.BuilderConfig	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : Optional[datasets.Features]      		=None






class 					snake_case					(   datasets.ArrowBasedBuilder	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : int      		=ArrowConfig






 def 	_lowerCamelCase (	self	:							List[str]     ):
        return datasets.DatasetInfo(features=self.config.features     )






 def 	_lowerCamelCase (	self	:							Union[str, Any]							,     __A	:							Dict     ):
        if not self.config.data_files:
               raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''     )
        __UpperCamelCase	=  dl_manager.download_and_extract(self.config.data_files     )
        if isinstance(__A							,     (str, list, tuple)     ):
               __UpperCamelCase	=  data_files
               if isinstance(__A							,     __A     ):
                      __UpperCamelCase	=  [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __UpperCamelCase	=  [dl_manager.iter_files(__A     ) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,     gen_kwargs={'files': files}     )]
        __UpperCamelCase	=  []
        for split_name, files in data_files.items():
               if isinstance(__A							,     __A     ):
                      __UpperCamelCase	=  [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __UpperCamelCase	=  [dl_manager.iter_files(__A     ) for file in files]
               # Infer features is they are stoed in the arrow schema
               if self.info.features is None:
                      for file in itertools.chain.from_iterable(__A     ):
                             with open(__A							,     'rb'     ) as f:
                                    __UpperCamelCase	=  datasets.Features.from_arrow_schema(pa.ipc.open_stream(__A     ).schema     )
                             break
               splits.append(datasets.SplitGenerator(name=__A							,     gen_kwargs={'files': files}     )     )
        return splits






 def 	_lowerCamelCase (	self	:							Any							,     __A	:							pa.Table     ):
        if self.info.features is not None:
               # more expensive cast to support nested features with keys in a different order
               # allows str <-> int/float or str to Audio for example
               __UpperCamelCase	=  table_cast(__A							,     self.info.features.arrow_schema     )
        return pa_table






 def 	_lowerCamelCase (	self	:							int							,     __A	:							Optional[Any]     ):
        for file_idx, file in enumerate(itertools.chain.from_iterable(__A     )     ):
               with open(__A							,     'rb'     ) as f:
                      try:
                             for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__A     )     ):
                                    __UpperCamelCase	=  pa.Table.from_batches([record_batch]     )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield f'''{file_idx}_{batch_idx}''', self._cast_table(__A     )
                      except ValueError as e:
                             logger.error(f'''Failed to read file \'{file}\' with error {type(__A     )}: {e}'''     )
                             raise
