'''simple docstring'''




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class      lowerCamelCase_     (snake_case__   ):
      '''simple docstring'''

      def __init__(	self						:    str						, A						:    NestedDataStructureLike[PathLike]						, A						:    Optional[NamedSplit] = None						, A						:    Optional[Features] = None						, A						:    str = None						, A						:    bool = False						, A						:    bool = False						, A						:    Optional[int] = None						, **A						:    Union[str, Any]						, ):
             super().__init__(
                 A						, split=A						, features=A						, cache_dir=A						, keep_in_memory=A						, streaming=A						, num_proc=A						, **A						, )
             _UpperCAmelCase					:  Optional[Any]											=   path_or_paths if isinstance(A						, A	) else {self.split: path_or_paths}
             _UpperCAmelCase					:  Optional[int]											=   Csv(
                 cache_dir=A						, data_files=A						, features=A						, **A						, )





      def 					_A     (	self						:    Tuple	):
             # Build iterable dataset
             if self.streaming:
                    _UpperCAmelCase					:  Optional[int]											=   self.builder.as_streaming_dataset(split=self.split	)
             # Build regular (map-style) dataset
             else:
                    _UpperCAmelCase					:  str											=   None
                    _UpperCAmelCase					:  Dict											=   None
                    _UpperCAmelCase					:  Optional[int]											=   None
                    _UpperCAmelCase					:  List[str]											=   None

                    self.builder.download_and_prepare(
                        download_config=A						, download_mode=A						, verification_mode=A						, base_path=A						, num_proc=self.num_proc						, )
                    _UpperCAmelCase					:  Union[str, Any]											=   self.builder.as_dataset(
                        split=self.split						, verification_mode=A						, in_memory=self.keep_in_memory	)
             return dataset

class      lowerCamelCase_     :
      '''simple docstring'''

      def __init__(	self						:    int						, A						:    Dataset						, A						:    Union[PathLike, BinaryIO]						, A						:    Optional[int] = None						, A						:    Optional[int] = None						, **A						:    Any						, ):
             if num_proc is not None and num_proc <= 0:
                    raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""	)

             _UpperCAmelCase					:  str											=   dataset
             _UpperCAmelCase					:  Union[str, Any]											=   path_or_buf
             _UpperCAmelCase					:  List[str]											=   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
             _UpperCAmelCase					:  Union[str, Any]											=   num_proc
             _UpperCAmelCase					:  Tuple											=   "utf-8"
             _UpperCAmelCase					:  Dict											=   to_csv_kwargs




      def 					_A     (	self						:    Union[str, Any]	):
             _UpperCAmelCase					:  Tuple											=   self.to_csv_kwargs.pop("path_or_buf"						, A	)
             _UpperCAmelCase					:  Optional[int]											=   self.to_csv_kwargs.pop("header"						, A	)
             _UpperCAmelCase					:  Union[str, Any]											=   self.to_csv_kwargs.pop("index"						, A	)

             if isinstance(self.path_or_buf						, (str, bytes, os.PathLike)	):
                    with open(self.path_or_buf						, "wb+"	) as buffer:
                           _UpperCAmelCase					:  Any											=   self._write(file_obj=A						, header=A						, index=A						, **self.to_csv_kwargs	)
             else:
                    _UpperCAmelCase					:  int											=   self._write(file_obj=self.path_or_buf						, header=A						, index=A						, **self.to_csv_kwargs	)
             return written




      def 					_A     (	self						:    Dict						, A						:    List[str]	):
             _UpperCAmelCase				,  _UpperCAmelCase				,  _UpperCAmelCase				,  _UpperCAmelCase					:  Optional[Any]											=   args

             _UpperCAmelCase					:  Union[str, Any]											=   query_table(
                 table=self.dataset.data						, key=slice(A						, offset + self.batch_size	)						, indices=self.dataset._indices						, )
             _UpperCAmelCase					:  Tuple											=   batch.to_pandas().to_csv(
                 path_or_buf=A						, header=header if (offset == 0) else False						, index=A						, **A	)
             return csv_str.encode(self.encoding	)





      def 					_A     (	self						:    List[Any]						, A						:    BinaryIO						, A						:    Dict						, A						:    Tuple						, **A						:    Optional[Any]	):
             _UpperCAmelCase					:  Dict											=   0

             if self.num_proc is None or self.num_proc == 1:
                    for offset in logging.tqdm(
                        range(0						, len(self.dataset	)						, self.batch_size	)						, unit="ba"						, disable=not logging.is_progress_bar_enabled()						, desc="Creating CSV from Arrow format"						, ):
                           _UpperCAmelCase					:  List[str]											=   self._batch_csv((offset, header, index, to_csv_kwargs)	)
                           written += file_obj.write(A	)

             else:
                    _UpperCAmelCase				,  _UpperCAmelCase					:  Dict											=   len(self.dataset	), self.batch_size
                    with multiprocessing.Pool(self.num_proc	) as pool:
                           for csv_str in logging.tqdm(
                               pool.imap(
                                   self._batch_csv						, [(offset, header, index, to_csv_kwargs) for offset in range(0						, A						, A	)]						, )						, total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						, unit="ba"						, disable=not logging.is_progress_bar_enabled()						, desc="Creating CSV from Arrow format"						, ):
                                  written += file_obj.write(A	)

             return written


'''simple docstring'''




import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter

class      lowerCamelCase_     (TensorFormatter[Mapping, np.ndarray, Mapping]   ):
      '''simple docstring'''

      def __init__(	self						:    str						, A						:    List[str]=None						, **A						:    Optional[Any]	):
             super().__init__(features=A	)
             _UpperCAmelCase					:  Tuple											=   np_array_kwargs




      def 					_A     (	self						:    str						, A						:    Tuple	):
             if isinstance(A						, A	):
                    if column and all(
                        isinstance(A						, np.ndarray	) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column	):
                           return np.stack(A	)
                    else:
                           # don't use np.array(column, dtype=object)
                           # since it fails in certain cases
                           # see https://stackoverflow.com/q/51005699
                           _UpperCAmelCase					:  List[str]											=   np.empty(len(A	)						, dtype=A	)
                           _UpperCAmelCase					:  int											=   column
                           return out
             return column




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             if isinstance(A						, (str, bytes, type(A	))	):
                    return value
             elif isinstance(A						, (np.character, np.ndarray)	) and np.issubdtype(value.dtype						, np.character	):
                    return value
             elif isinstance(A						, np.number	):
                    return value

             _UpperCAmelCase					:  List[str]											=   {}

             if isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.integer	):
                    _UpperCAmelCase					:  str											=   {"dtype": np.intaa}
             elif isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.floating	):
                    _UpperCAmelCase					:  int											=   {"dtype": np.floataa}
             elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                    import PIL.Image

                    if isinstance(A						, PIL.Image.Image	):
                           return np.asarray(A						, **self.np_array_kwargs	)

             return np.asarray(A						, **{**default_dtype, **self.np_array_kwargs}	)




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             # support for torch, tf, jax etc.
             if config.TORCH_AVAILABLE and "torch" in sys.modules:
                    import torch

                    if isinstance(A						, torch.Tensor	):
                           return self._tensorize(data_struct.detach().cpu().numpy()[()]	)
             if hasattr(A						, "__array__"	) and not isinstance(A						, (np.ndarray, np.character, np.number)	):
                    _UpperCAmelCase					:  Dict											=   data_struct.__array__()
             # support for nested types like struct of list of struct
             if isinstance(A						, np.ndarray	):
                    if data_struct.dtype == object:
                           return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             if isinstance(A						, (list, tuple)	):
                    return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             return self._tensorize(A	)




      def 					_A     (	self						:    List[Any]						, A						:    dict	):
             return map_nested(self._recursive_tensorize						, A						, map_list=A	)




      def 					_A     (	self						:    List[str]						, A						:    pa.Table	):
             _UpperCAmelCase					:  str											=   self.numpy_arrow_extractor().extract_row(A	)
             _UpperCAmelCase					:  Dict											=   self.python_features_decoder.decode_row(A	)
             return self.recursive_tensorize(A	)




      def 					_A     (	self						:    str						, A						:    pa.Table	):
             _UpperCAmelCase					:  List[str]											=   self.numpy_arrow_extractor().extract_column(A	)
             _UpperCAmelCase					:  Optional[Any]											=   self.python_features_decoder.decode_column(A						, pa_table.column_names[0]	)
             _UpperCAmelCase					:  Tuple											=   self.recursive_tensorize(A	)
             _UpperCAmelCase					:  Tuple											=   self._consolidate(A	)
             return column





      def 					_A     (	self						:    Union[str, Any]						, A						:    pa.Table	):
             _UpperCAmelCase					:  Any											=   self.numpy_arrow_extractor().extract_batch(A	)
             _UpperCAmelCase					:  Any											=   self.python_features_decoder.decode_batch(A	)
             _UpperCAmelCase					:  List[str]											=   self.recursive_tensorize(A	)
             for column_name in batch:
                    _UpperCAmelCase					:  Optional[Any]											=   self._consolidate(batch[column_name]	)
             return batch

