




'''simple docstring'''






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				def __init__(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None			,  _UpperCAmelCase = None			,  _UpperCAmelCase = None			,  _UpperCAmelCase = False			,  _UpperCAmelCase = False			,  _UpperCAmelCase = None			,  **_UpperCAmelCase			,  )     ->	str:
							super().__init__(
							    _UpperCAmelCase			,  split=_UpperCAmelCase			,  features=_UpperCAmelCase			,  cache_dir=_UpperCAmelCase			,  keep_in_memory=_UpperCAmelCase			,  streaming=_UpperCAmelCase			,  num_proc=_UpperCAmelCase			,  **_UpperCAmelCase			,  )
							__UpperCamelCase				:      Tuple									=       path_or_paths if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else {self.split: path_or_paths}
							__UpperCamelCase				:      Union[str, Any]									=       Csv(
							    cache_dir=_UpperCAmelCase			,  data_files=_UpperCAmelCase			,  features=_UpperCAmelCase			,  **_UpperCAmelCase			,  )





				def 	a_	(self      )     ->	Optional[int]:
							# Build iterable dataset
							if self.streaming:
										__UpperCamelCase				:      Union[str, Any]									=       self.builder.as_streaming_dataset(split=self.split      )
							# Build regular (map-style) dataset
							else:
										__UpperCamelCase				:      Tuple									=       None
										__UpperCamelCase				:      str									=       None
										__UpperCamelCase				:      Union[str, Any]									=       None
										__UpperCamelCase				:      List[str]									=       None

										self.builder.download_and_prepare(
										    download_config=_UpperCAmelCase			,  download_mode=_UpperCAmelCase			,  verification_mode=_UpperCAmelCase			,  base_path=_UpperCAmelCase			,  num_proc=self.num_proc			,  )
										__UpperCamelCase				:      Optional[int]									=       self.builder.as_dataset(
										    split=self.split			,  verification_mode=_UpperCAmelCase			,  in_memory=self.keep_in_memory      )
							return dataset







class  A		:




				'''simple docstring'''





				def __init__(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase = None			,  _UpperCAmelCase = None			,  **_UpperCAmelCase			,  )     ->	Dict:
							if num_proc is not None and num_proc <= 0:
										raise ValueError(f"num_proc {num_proc} must be an integer > 0."      )

							__UpperCamelCase				:      List[Any]									=       dataset
							__UpperCamelCase				:      int									=       path_or_buf
							__UpperCamelCase				:      Optional[int]									=       batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
							__UpperCamelCase				:      Union[str, Any]									=       num_proc
							__UpperCamelCase				:      Optional[Any]									=       "utf-8"
							__UpperCamelCase				:      Optional[Any]									=       to_csv_kwargs
				def 	a_	(self      )     ->	int:
							__UpperCamelCase				:      Any									=       self.to_csv_kwargs.pop("path_or_buf"			,  _UpperCAmelCase      )
							__UpperCamelCase				:      int									=       self.to_csv_kwargs.pop("header"			,  _UpperCAmelCase      )
							__UpperCamelCase				:      int									=       self.to_csv_kwargs.pop("index"			,  _UpperCAmelCase      )

							if isinstance(self.path_or_buf			,  (str, bytes, os.PathLike)      ):
										with open(self.path_or_buf			,  "wb+"      ) as buffer:
													__UpperCamelCase				:      Optional[Any]									=       self._write(file_obj=_UpperCAmelCase			,  header=_UpperCAmelCase			,  index=_UpperCAmelCase			,  **self.to_csv_kwargs      )
							else:
										__UpperCamelCase				:      Optional[Any]									=       self._write(file_obj=self.path_or_buf			,  header=_UpperCAmelCase			,  index=_UpperCAmelCase			,  **self.to_csv_kwargs      )
							return written
				def 	a_	(self			,  _UpperCAmelCase      )     ->	List[Any]:
							__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase				:      Dict									=       args

							__UpperCamelCase				:      Any									=       query_table(
							    table=self.dataset.data			,  key=slice(_UpperCAmelCase			,  offset + self.batch_size      )			,  indices=self.dataset._indices			,  )
							__UpperCamelCase				:      int									=       batch.to_pandas().to_csv(
							    path_or_buf=_UpperCAmelCase			,  header=header if (offset == 0) else False			,  index=_UpperCAmelCase			,  **_UpperCAmelCase      )
							return csv_str.encode(self.encoding      )





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase			,  **_UpperCAmelCase      )     ->	int:
							__UpperCamelCase				:      Tuple									=       0

							if self.num_proc is None or self.num_proc == 1:
										for offset in logging.tqdm(
										    range(0			,  len(self.dataset      )			,  self.batch_size      )			,  unit="ba"			,  disable=not logging.is_progress_bar_enabled()			,  desc="Creating CSV from Arrow format"			,  ):
													__UpperCamelCase				:      Union[str, Any]									=       self._batch_csv((offset, header, index, to_csv_kwargs)      )
													written += file_obj.write(_UpperCAmelCase      )

							else:
										__UpperCamelCase		, __UpperCamelCase				:      List[Any]									=       len(self.dataset      ), self.batch_size
										with multiprocessing.Pool(self.num_proc      ) as pool:
													for csv_str in logging.tqdm(
													    pool.imap(
													        self._batch_csv			,  [(offset, header, index, to_csv_kwargs) for offset in range(0			,  _UpperCAmelCase			,  _UpperCAmelCase      )]			,  )			,  total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,  unit="ba"			,  disable=not logging.is_progress_bar_enabled()			,  desc="Creating CSV from Arrow format"			,  ):
																written += file_obj.write(_UpperCAmelCase      )

							return written









'''simple docstring'''






from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_lowerCAmelCase				     = logging.get_logger(__name__)

_lowerCAmelCase				     = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase				     = {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
    '''tokenizer_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/tokenizer.json'''},
}

_lowerCAmelCase				     = {
    '''openai-gpt''': 512,
}



class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				A				     =    VOCAB_FILES_NAMES
				A				     =    PRETRAINED_VOCAB_FILES_MAP
				A				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A				     =    ["input_ids", "attention_mask"]
				A				     =    OpenAIGPTTokenizer
				def __init__(self			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase="<unk>"			,  **_UpperCAmelCase      )     ->	str:
							super().__init__(_UpperCAmelCase			,  _UpperCAmelCase			,  tokenizer_file=_UpperCAmelCase			,  unk_token=_UpperCAmelCase			,  **_UpperCAmelCase      )
				@property
				def 	a_	(self      )     ->	Dict:
							return True





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	Tuple[str]:
							__UpperCamelCase				:      Optional[Any]									=       self._tokenizer.model.save(_UpperCAmelCase			,  name=_UpperCAmelCase      )
							return tuple(_UpperCAmelCase      )



