




import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter
class    __lowerCAmelCase			(	TensorFormatter[Mapping, np.ndarray, Mapping]			):







			def __init__(	self   :      Dict   ,   snake_case__   :      Union[str, Any]=None   ,   **snake_case__   :      Tuple				):




									"""simple docstring"""


									super().__init__(features=UpperCamelCase__				)
									_UpperCAmelCase													=					np_array_kwargs







			def UpperCamelCase      (	self   :      Optional[Any]   ,   snake_case__   :      Optional[int]				):




									"""simple docstring"""


									if isinstance(UpperCamelCase__   ,   UpperCamelCase__				):
															if column and all(
															    isinstance(UpperCamelCase__   ,   np.ndarray				) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column				):
																					return np.stack(UpperCamelCase__				)
															else:
																					# don't use np.array(column, dtype=object)
																					# since it fails in certain cases
																					# see https://stackoverflow.com/q/51005699
																					_UpperCAmelCase													=					np.empty(len(UpperCamelCase__				)   ,   dtype=UpperCamelCase__				)
																					_UpperCAmelCase													=					column
																					return out
									return column







			def UpperCamelCase      (	self   :      str   ,   snake_case__   :      Any				):




									"""simple docstring"""


									if isinstance(UpperCamelCase__   ,   (str, bytes, type(UpperCamelCase__				))				):
															return value
									elif isinstance(UpperCamelCase__   ,   (np.character, np.ndarray)				) and np.issubdtype(value.dtype   ,   np.character				):
															return value
									elif isinstance(UpperCamelCase__   ,   np.number				):
															return value

									_UpperCAmelCase													=					{}

									if isinstance(UpperCamelCase__   ,   np.ndarray				) and np.issubdtype(value.dtype   ,   np.integer				):
															_UpperCAmelCase													=					{"dtype": np.intaa}
									elif isinstance(UpperCamelCase__   ,   np.ndarray				) and np.issubdtype(value.dtype   ,   np.floating				):
															_UpperCAmelCase													=					{"dtype": np.floataa}
									elif config.PIL_AVAILABLE and "PIL" in sys.modules:
															import PIL.Image

															if isinstance(UpperCamelCase__   ,   PIL.Image.Image				):
																					return np.asarray(UpperCamelCase__   ,   **self.np_array_kwargs				)

									return np.asarray(UpperCamelCase__   ,   **{**default_dtype, **self.np_array_kwargs}				)







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      int				):




									"""simple docstring"""


									if config.TORCH_AVAILABLE and "torch" in sys.modules:
															import torch

															if isinstance(UpperCamelCase__   ,   torch.Tensor				):
																					return self._tensorize(data_struct.detach().cpu().numpy()[()]				)
									if hasattr(UpperCamelCase__   ,   "__array__"				) and not isinstance(UpperCamelCase__   ,   (np.ndarray, np.character, np.number)				):
															_UpperCAmelCase													=					data_struct.__array__()
									# support for nested types like struct of list of struct
									if isinstance(UpperCamelCase__   ,   np.ndarray				):
															if data_struct.dtype == object:
																					return self._consolidate([self.recursive_tensorize(UpperCamelCase__				) for substruct in data_struct]				)
									if isinstance(UpperCamelCase__   ,   (list, tuple)				):
															return self._consolidate([self.recursive_tensorize(UpperCamelCase__				) for substruct in data_struct]				)
									return self._tensorize(UpperCamelCase__				)







			def UpperCamelCase      (	self   :      Dict   ,   snake_case__   :      List[str]				):




									"""simple docstring"""


									return map_nested(self._recursive_tensorize   ,   UpperCamelCase__   ,   map_list=UpperCamelCase__				)







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      Any				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.numpy_arrow_extractor().extract_row(UpperCamelCase__				)
									_UpperCAmelCase													=					self.python_features_decoder.decode_row(UpperCamelCase__				)
									return self.recursive_tensorize(UpperCamelCase__				)







			def UpperCamelCase      (	self   :      Optional[int]   ,   snake_case__   :      Dict				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.numpy_arrow_extractor().extract_column(UpperCamelCase__				)
									_UpperCAmelCase													=					self.python_features_decoder.decode_column(UpperCamelCase__   ,   pa_table.column_names[0]				)
									_UpperCAmelCase													=					self.recursive_tensorize(UpperCamelCase__				)
									_UpperCAmelCase													=					self._consolidate(UpperCamelCase__				)
									return column







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      Optional[int]				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.numpy_arrow_extractor().extract_batch(UpperCamelCase__				)
									_UpperCAmelCase													=					self.python_features_decoder.decode_batch(UpperCamelCase__				)
									_UpperCAmelCase													=					self.recursive_tensorize(UpperCamelCase__				)
									for column_name in batch:
															_UpperCAmelCase													=					self._consolidate(batch[column_name]				)
									return batch










import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowercase_     :    Union[str, Any]      =			datasets.utils.logging.get_logger(__name__)






@dataclass
class    __lowerCAmelCase			(	datasets.BuilderConfig			):
			snake_case_				:			Optional[datasets.Features]								=					None






class    __lowerCAmelCase			(	datasets.ArrowBasedBuilder			):
			snake_case_				:			str								=					ArrowConfig







			def UpperCamelCase      (	self   :      Optional[int]				):




									"""simple docstring"""


									return datasets.DatasetInfo(features=self.config.features				)







			def UpperCamelCase      (	self   :      str   ,   snake_case__   :      Optional[int]				):




									"""simple docstring"""


									if not self.config.data_files:
															raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""				)
									_UpperCAmelCase													=					dl_manager.download_and_extract(self.config.data_files				)
									if isinstance(snake_case__   ,   (str, list, tuple)				):
															_UpperCAmelCase													=					data_files
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,   gen_kwargs={"files": files}				)]
									_UpperCAmelCase													=					[]
									for split_name, files in data_files.items():
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															# Infer features is they are stoed in the arrow schema
															if self.info.features is None:
																					for file in itertools.chain.from_iterable(snake_case__				):
																											with open(snake_case__   ,   "rb"				) as f:
																																	_UpperCAmelCase													=					datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case__				).schema				)
																											break
															splits.append(datasets.SplitGenerator(name=snake_case__   ,   gen_kwargs={"files": files}				)				)
									return splits







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      pa.Table				):




									"""simple docstring"""


									if self.info.features is not None:
															# more expensive cast to support nested features with keys in a different order
															# allows str <-> int/float or str to Audio for example
															_UpperCAmelCase													=					table_cast(snake_case__   ,   self.info.features.arrow_schema				)
									return pa_table







			def UpperCamelCase      (	self   :      Optional[Any]   ,   snake_case__   :      int				):




									"""simple docstring"""


									for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__				)				):
															with open(snake_case__   ,   "rb"				) as f:
																					try:
																											for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case__				)				):
																																	_UpperCAmelCase													=					pa.Table.from_batches([record_batch]				)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield F"""{file_idx}_{batch_idx}""", self._cast_table(snake_case__				)
																					except ValueError as e:
																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case__				)}: {e}"""				)
																											raise




