




import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowercase_     :    str      =			datasets.utils.logging.get_logger(__name__)






@dataclass
class    __lowerCAmelCase			(	datasets.BuilderConfig			):
			snake_case_				:			Optional[datasets.Features]								=					None
			snake_case_				:			str								=					"utf-8"
			snake_case_				:			Optional[str]								=					None
			snake_case_				:			int								=					10 << 20  # 10MB
			snake_case_				:			bool								=					False
			snake_case_				:			str								=					"line"






class    __lowerCAmelCase			(	datasets.ArrowBasedBuilder			):
			snake_case_				:			str								=					TextConfig







			def UpperCamelCase      (	self   :      Optional[Any]				):




									"""simple docstring"""


									return datasets.DatasetInfo(features=self.config.features				)







			def UpperCamelCase      (	self   :      Optional[int]   ,   snake_case__   :      Optional[int]				):




									"""simple docstring"""


									if not self.config.data_files:
															raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""				)
									_UpperCAmelCase													=					dl_manager.download_and_extract(self.config.data_files				)
									if isinstance(snake_case__   ,   (str, list, tuple)				):
															_UpperCAmelCase													=					data_files
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,   gen_kwargs={"files": files}				)]
									_UpperCAmelCase													=					[]
									for split_name, files in data_files.items():
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															splits.append(datasets.SplitGenerator(name=snake_case__   ,   gen_kwargs={"files": files}				)				)
									return splits







			def UpperCamelCase      (	self   :      Optional[int]   ,   snake_case__   :      pa.Table				):




									"""simple docstring"""


									if self.config.features is not None:
															_UpperCAmelCase													=					self.config.features.arrow_schema
															if all(not require_storage_cast(snake_case__				) for feature in self.config.features.values()				):
																					# cheaper cast
																					_UpperCAmelCase													=					pa_table.cast(snake_case__				)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					_UpperCAmelCase													=					table_cast(snake_case__   ,   snake_case__				)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}				)				)







			def UpperCamelCase      (	self   :      Tuple   ,   snake_case__   :      Any				):




									"""simple docstring"""


									_UpperCAmelCase													=					list(self.config.features				) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__				)				):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(snake_case__   ,   encoding=self.config.encoding   ,   errors=self.config.errors				) as f:
																					if self.config.sample_by == "line":
																											_UpperCAmelCase													=					0
																											while True:
																																	_UpperCAmelCase													=					f.read(self.config.chunksize				)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	_UpperCAmelCase													=					StringIO(snake_case__				).readlines()
																																	if not self.config.keep_linebreaks:
																																							_UpperCAmelCase													=					[line.rstrip("\n"				) for line in batch]
																																	_UpperCAmelCase													=					pa.Table.from_arrays([pa.array(snake_case__				)]   ,   names=snake_case__				)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(snake_case__				)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											_UpperCAmelCase													=					0
																											_UpperCAmelCase													=					""
																											while True:
																																	_UpperCAmelCase													=					f.read(self.config.chunksize				)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	_UpperCAmelCase													=					batch.split("\n\n"				)
																																	_UpperCAmelCase													=					pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example]				)]   ,   names=snake_case__				)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(snake_case__				)
																																	batch_idx += 1
																																	_UpperCAmelCase													=					batch[-1]
																											if batch:
																																	_UpperCAmelCase													=					pa.Table.from_arrays([pa.array([batch]				)]   ,   names=snake_case__				)
																																	yield (file_idx, batch_idx), self._cast_table(snake_case__				)
																					elif self.config.sample_by == "document":
																											_UpperCAmelCase													=					f.read()
																											_UpperCAmelCase													=					pa.Table.from_arrays([pa.array([text]				)]   ,   names=snake_case__				)
																											yield file_idx, self._cast_table(snake_case__				)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase_     :    int      =			{
    'configuration_videomae': ['VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VideoMAEConfig'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_     :    List[str]      =			[
				    'VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'VideoMAEForPreTraining',
				    'VideoMAEModel',
				    'VideoMAEPreTrainedModel',
				    'VideoMAEForVideoClassification',
				]

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_     :    Any      =			['VideoMAEFeatureExtractor']
				lowercase_     :    Dict      =			['VideoMAEImageProcessor']

if TYPE_CHECKING:
				from .configuration_videomae import VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP, VideoMAEConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_videomae import (
								    VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST,
								    VideoMAEForPreTraining,
								    VideoMAEForVideoClassification,
								    VideoMAEModel,
								    VideoMAEPreTrainedModel,
								)

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_videomae import VideoMAEFeatureExtractor
								from .image_processing_videomae import VideoMAEImageProcessor

else:
				import sys

				lowercase_     :    int      =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




