






"""simple docstring"""




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class 					__SCREAMING_SNAKE_CASE(		a_		):
							def __init__(       self: Any    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: List[str] = None    ,		UpperCamelCase: Tuple = None    ,		UpperCamelCase: List[Any] = None    ,		UpperCamelCase: Optional[int] = False    ,		UpperCamelCase: str = False    ,		UpperCamelCase: Dict = None    ,		**UpperCamelCase: Dict    ,		)						->						Optional[int]:
											super().__init__(
											    __A    ,		split=__A    ,		features=__A    ,		cache_dir=__A    ,		keep_in_memory=__A    ,		streaming=__A    ,		num_proc=__A    ,		**__A    ,		)
											snake_case__													=       path_or_paths if isinstance(__A    ,		__A  ) else {self.split: path_or_paths}
											snake_case__													=       Csv(
											    cache_dir=__A    ,		data_files=__A    ,		features=__A    ,		**__A    ,		)


							def lowerCAmelCase_	(       self: List[str]  )						->						Optional[int]:





											if self.streaming:
															snake_case__													=       self.builder.as_streaming_dataset(split=self.split  )
											# Build regular (map-style) dataset
											else:
															snake_case__													=       None
															snake_case__													=       None
															snake_case__													=       None
															snake_case__													=       None

															self.builder.download_and_prepare(
															    download_config=__A    ,		download_mode=__A    ,		verification_mode=__A    ,		base_path=__A    ,		num_proc=self.num_proc    ,		)
															snake_case__													=       self.builder.as_dataset(
															    split=self.split    ,		verification_mode=__A    ,		in_memory=self.keep_in_memory  )
											return dataset


class 					__SCREAMING_SNAKE_CASE:
							def __init__(       self: Tuple    ,		UpperCamelCase: Any    ,		UpperCamelCase: Dict    ,		UpperCamelCase: Tuple = None    ,		UpperCamelCase: Dict = None    ,		**UpperCamelCase: str    ,		)						->						Any:





											if num_proc is not None and num_proc <= 0:
															raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''  )

											snake_case__													=       dataset
											snake_case__													=       path_or_buf
											snake_case__													=       batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											snake_case__													=       num_proc
											snake_case__													=       "utf-8"
											snake_case__													=       to_csv_kwargs
							def lowerCAmelCase_	(       self: int  )						->						List[str]:
											snake_case__													=       self.to_csv_kwargs.pop('path_or_buf'    ,		__A  )
											snake_case__													=       self.to_csv_kwargs.pop('header'    ,		__A  )
											snake_case__													=       self.to_csv_kwargs.pop('index'    ,		__A  )

											if isinstance(self.path_or_buf    ,		(str, bytes, os.PathLike)  ):
															with open(self.path_or_buf    ,		'wb+'  ) as buffer:
																			snake_case__													=       self._write(file_obj=__A    ,		header=__A    ,		index=__A    ,		**self.to_csv_kwargs  )
											else:
															snake_case__													=       self._write(file_obj=self.path_or_buf    ,		header=__A    ,		index=__A    ,		**self.to_csv_kwargs  )
											return written
							def lowerCAmelCase_	(       self: List[str]    ,		UpperCamelCase: Dict  )						->						Tuple:
											snake_case__													=       args

											snake_case__													=       query_table(
											    table=self.dataset.data    ,		key=slice(__A    ,		offset + self.batch_size  )    ,		indices=self.dataset._indices    ,		)
											snake_case__													=       batch.to_pandas().to_csv(
											    path_or_buf=__A    ,		header=header if (offset == 0) else False    ,		index=__A    ,		**__A  )
											return csv_str.encode(self.encoding  )


							def lowerCAmelCase_	(       self: int    ,		UpperCamelCase: Any    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: List[Any]    ,		**UpperCamelCase: Any  )						->						int:
											snake_case__													=       0

											if self.num_proc is None or self.num_proc == 1:
															for offset in logging.tqdm(
															    range(0    ,		len(self.dataset  )    ,		self.batch_size  )    ,		unit='ba'    ,		disable=not logging.is_progress_bar_enabled()    ,		desc='Creating CSV from Arrow format'    ,		):
																			snake_case__													=       self._batch_csv((offset, header, index, to_csv_kwargs)  )
																			written += file_obj.write(__A  )

											else:
															snake_case__													=       len(self.dataset  ), self.batch_size
															with multiprocessing.Pool(self.num_proc  ) as pool:
																			for csv_str in logging.tqdm(
																			    pool.imap(
																			        self._batch_csv    ,		[(offset, header, index, to_csv_kwargs) for offset in range(0    ,		__A    ,		__A  )]    ,		)    ,		total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size    ,		unit='ba'    ,		disable=not logging.is_progress_bar_enabled()    ,		desc='Creating CSV from Arrow format'    ,		):
																							written += file_obj.write(__A  )

											return written












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase   :						int							=  {
    """configuration_openai""": ["""OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OpenAIGPTConfig"""],
    """tokenization_openai""": ["""OpenAIGPTTokenizer"""],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						Optional[Any]							=  ["""OpenAIGPTTokenizerFast"""]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						List[str]							=  [
							    """OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """OpenAIGPTDoubleHeadsModel""",
							    """OpenAIGPTForSequenceClassification""",
							    """OpenAIGPTLMHeadModel""",
							    """OpenAIGPTModel""",
							    """OpenAIGPTPreTrainedModel""",
							    """load_tf_weights_in_openai_gpt""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						Dict							=  [
							    """TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFOpenAIGPTDoubleHeadsModel""",
							    """TFOpenAIGPTForSequenceClassification""",
							    """TFOpenAIGPTLMHeadModel""",
							    """TFOpenAIGPTMainLayer""",
							    """TFOpenAIGPTModel""",
							    """TFOpenAIGPTPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
							from .tokenization_openai import OpenAIGPTTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_openai_fast import OpenAIGPTTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_openai import (
														    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    OpenAIGPTDoubleHeadsModel,
														    OpenAIGPTForSequenceClassification,
														    OpenAIGPTLMHeadModel,
														    OpenAIGPTModel,
														    OpenAIGPTPreTrainedModel,
														    load_tf_weights_in_openai_gpt,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_openai import (
														    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFOpenAIGPTDoubleHeadsModel,
														    TFOpenAIGPTForSequenceClassification,
														    TFOpenAIGPTLMHeadModel,
														    TFOpenAIGPTMainLayer,
														    TFOpenAIGPTModel,
														    TFOpenAIGPTPreTrainedModel,
														)

else:
							import sys

							__UpperCamelCase   :						List[Any]							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




