



'''simple docstring'''




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 						_UpperCamelCase  (							_a   ):


			'''simple docstring'''

			def __init__(							self					:			Optional[Any]							,					lowerCAmelCase__					:			NestedDataStructureLike[PathLike]							,					lowerCAmelCase__					:			Optional[NamedSplit] = None							,					lowerCAmelCase__					:			Optional[Features] = None							,					lowerCAmelCase__					:			str = None							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			Optional[int] = None							,					**lowerCAmelCase__					:			int							,					):





					"""simple docstring"""






					super().__init__(
					    SCREAMING_SNAKE_CASE__							,					split=SCREAMING_SNAKE_CASE__							,					features=SCREAMING_SNAKE_CASE__							,					cache_dir=SCREAMING_SNAKE_CASE__							,					keep_in_memory=SCREAMING_SNAKE_CASE__							,					streaming=SCREAMING_SNAKE_CASE__							,					num_proc=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)
					__SCREAMING_SNAKE_CASE	:		str   	=		path_or_paths if isinstance(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__						) else {self.split: path_or_paths}
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		Csv(
					    cache_dir=SCREAMING_SNAKE_CASE__							,					data_files=SCREAMING_SNAKE_CASE__							,					features=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)



			def UpperCamelCase__ (							self					:			str						):





					"""simple docstring"""






					if self.streaming:
							__SCREAMING_SNAKE_CASE	:		Tuple   	=		self.builder.as_streaming_dataset(split=self.split						)
					# Build regular (map-style) dataset
					else:
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		None
							__SCREAMING_SNAKE_CASE	:		str   	=		None
							__SCREAMING_SNAKE_CASE	:		Dict   	=		None
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		None

							self.builder.download_and_prepare(
							    download_config=SCREAMING_SNAKE_CASE__							,					download_mode=SCREAMING_SNAKE_CASE__							,					verification_mode=SCREAMING_SNAKE_CASE__							,					base_path=SCREAMING_SNAKE_CASE__							,					num_proc=self.num_proc							,					)
							__SCREAMING_SNAKE_CASE	:		str   	=		self.builder.as_dataset(
							    split=self.split							,					verification_mode=SCREAMING_SNAKE_CASE__							,					in_memory=self.keep_in_memory						)
					return dataset






class 						_UpperCamelCase  :


			'''simple docstring'''

			def __init__(							self					:			Union[str, Any]							,					lowerCAmelCase__					:			Dataset							,					lowerCAmelCase__					:			Union[PathLike, BinaryIO]							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			Optional[int] = None							,					**lowerCAmelCase__					:			Any							,					):





					"""simple docstring"""






					if num_proc is not None and num_proc <= 0:
							raise ValueError(F"num_proc {num_proc} must be an integer > 0."						)

					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		dataset
					__SCREAMING_SNAKE_CASE	:		str   	=		path_or_buf
					__SCREAMING_SNAKE_CASE	:		Any   	=		batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
					__SCREAMING_SNAKE_CASE	:		int   	=		num_proc
					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		"utf-8"
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		to_csv_kwargs

			def UpperCamelCase__ (							self					:			int						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		int   	=		self.to_csv_kwargs.pop("""path_or_buf"""							,					SCREAMING_SNAKE_CASE__						)
					__SCREAMING_SNAKE_CASE	:		Dict   	=		self.to_csv_kwargs.pop("""header"""							,					SCREAMING_SNAKE_CASE__						)
					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		self.to_csv_kwargs.pop("""index"""							,					SCREAMING_SNAKE_CASE__						)

					if isinstance(self.path_or_buf							,					(str, bytes, os.PathLike)						):
							with open(self.path_or_buf							,					"""wb+"""						) as buffer:
									__SCREAMING_SNAKE_CASE	:		int   	=		self._write(file_obj=SCREAMING_SNAKE_CASE__							,					header=SCREAMING_SNAKE_CASE__							,					index=SCREAMING_SNAKE_CASE__							,					**self.to_csv_kwargs						)
					else:
							__SCREAMING_SNAKE_CASE	:		List[str]   	=		self._write(file_obj=self.path_or_buf							,					header=SCREAMING_SNAKE_CASE__							,					index=SCREAMING_SNAKE_CASE__							,					**self.to_csv_kwargs						)
					return written

			def UpperCamelCase__ (							self					:			List[str]							,					lowerCAmelCase__					:			Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		List[str]   	=		args

					__SCREAMING_SNAKE_CASE	:		str   	=		query_table(
					    table=self.dataset.data							,					key=slice(SCREAMING_SNAKE_CASE__							,					offset + self.batch_size						)							,					indices=self.dataset._indices							,					)
					__SCREAMING_SNAKE_CASE	:		Dict   	=		batch.to_pandas().to_csv(
					    path_or_buf=SCREAMING_SNAKE_CASE__							,					header=header if (offset == 0) else False							,					index=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__						)
					return csv_str.encode(self.encoding						)



			def UpperCamelCase__ (							self					:			int							,					lowerCAmelCase__					:			BinaryIO							,					lowerCAmelCase__					:			str							,					lowerCAmelCase__					:			Any							,					**lowerCAmelCase__					:			List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		0

					if self.num_proc is None or self.num_proc == 1:
							for offset in logging.tqdm(
							    range(0							,					len(self.dataset						)							,					self.batch_size						)							,					unit="""ba"""							,					disable=not logging.is_progress_bar_enabled()							,					desc="""Creating CSV from Arrow format"""							,					):
									__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		self._batch_csv((offset, header, index, to_csv_kwargs)						)
									written += file_obj.write(SCREAMING_SNAKE_CASE__						)

					else:
							__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		len(self.dataset						), self.batch_size
							with multiprocessing.Pool(self.num_proc						) as pool:
									for csv_str in logging.tqdm(
									    pool.imap(
									        self._batch_csv							,					[(offset, header, index, to_csv_kwargs) for offset in range(0							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__						)]							,					)							,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size							,					unit="""ba"""							,					disable=not logging.is_progress_bar_enabled()							,					desc="""Creating CSV from Arrow format"""							,					):
											written += file_obj.write(SCREAMING_SNAKE_CASE__						)

					return written




'''simple docstring'''




import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 						_UpperCamelCase  (							lowerCamelCase__   ):


			'''simple docstring'''

			_A				:		int         =		['''image_processor''', '''tokenizer''']
			_A				:		Union[str, Any]         =		'''AutoImageProcessor'''
			_A				:		Optional[Any]         =		'''AutoTokenizer'''

			def __init__(							self					:			Tuple							,					lowerCAmelCase__					:			Tuple=None							,					lowerCAmelCase__					:			Union[str, Any]=None							,					**lowerCAmelCase__					:			Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Tuple   	=		None
					if "feature_extractor" in kwargs:
							warnings.warn(
							    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
							    """ instead."""							,					lowerCAmelCase__							,					)
							__SCREAMING_SNAKE_CASE	:		str   	=		kwargs.pop("""feature_extractor"""						)

					__SCREAMING_SNAKE_CASE	:		Any   	=		image_processor if image_processor is not None else feature_extractor
					if image_processor is None:
							raise ValueError("""You need to specify an `image_processor`."""						)
					if tokenizer is None:
							raise ValueError("""You need to specify a `tokenizer`."""						)

					super().__init__(lowerCAmelCase__							,					lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE	:		List[str]   	=		self.image_processor
					__SCREAMING_SNAKE_CASE	:		Dict   	=		False

			def __call__(							self					:			List[str]							,					*lowerCAmelCase__					:			List[str]							,					**lowerCAmelCase__					:			Dict						):





					"""simple docstring"""






					if self._in_target_context_manager:
							return self.current_processor(*lowerCAmelCase__							,					**lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		kwargs.pop("""images"""							,					lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		kwargs.pop("""text"""							,					lowerCAmelCase__						)
					if len(lowerCAmelCase__						) > 0:
							__SCREAMING_SNAKE_CASE	:		Tuple   	=		args[0]
							__SCREAMING_SNAKE_CASE	:		int   	=		args[1:]

					if images is None and text is None:
							raise ValueError("""You need to specify either an `images` or `text` input to process."""						)

					if images is not None:
							__SCREAMING_SNAKE_CASE	:		List[str]   	=		self.image_processor(lowerCAmelCase__							,					*lowerCAmelCase__							,					**lowerCAmelCase__						)
					if text is not None:
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		self.tokenizer(lowerCAmelCase__							,					**lowerCAmelCase__						)

					if text is None:
							return inputs
					elif images is None:
							return encodings
					else:
							__SCREAMING_SNAKE_CASE	:		int   	=		encodings["""input_ids"""]
							return inputs

			def UpperCamelCase__ (							self					:			Any							,					*lowerCAmelCase__					:			Any							,					**lowerCAmelCase__					:			List[str]						):





					"""simple docstring"""






					return self.tokenizer.batch_decode(*lowerCAmelCase__							,					**lowerCAmelCase__						)

			def UpperCamelCase__ (							self					:			int							,					*lowerCAmelCase__					:			str							,					**lowerCAmelCase__					:			Dict						):





					"""simple docstring"""






					return self.tokenizer.decode(*lowerCAmelCase__							,					**lowerCAmelCase__						)

			@contextmanager
			def UpperCamelCase__ (							self					:			Union[str, Any]						):





					"""simple docstring"""






					warnings.warn(
					    """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
					    """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
					    """your images inputs, or in a separate call."""						)
					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		True
					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		self.tokenizer
					yield
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		self.image_processor
					__SCREAMING_SNAKE_CASE	:		List[str]   	=		False

			@property
			def UpperCamelCase__ (							self					:			Optional[Any]						):





					"""simple docstring"""






					warnings.warn(
					    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""							,					lowerCAmelCase__							,					)
					return self.image_processor_class



			@property
			def UpperCamelCase__ (							self					:			List[Any]						):





					"""simple docstring"""






					warnings.warn(
					    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""							,					lowerCAmelCase__							,					)
					return self.image_processor