



"""simple docstring"""
import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__a     =    datasets.utils.logging.get_logger(__name__)
@dataclass
class     lowerCamelCase  (      datasets.BuilderConfig    ):






					'''simple docstring'''





					_A		:      Optional[datasets.Features]						=					None
					_A		:      str						=					"utf-8"
					_A		:      Optional[str]						=					None
					_A		:      int						=					1_0 << 2_0  # 10MB
					_A		:      bool						=					False
					_A		:      str						=					"line"

class     lowerCamelCase  (      datasets.ArrowBasedBuilder    ):






					'''simple docstring'''





					_A		:      Union[str, Any]						=					TextConfig






					def        lowerCAmelCase_    (	self:							Tuple						)       ->     Optional[int]:
											return datasets.DatasetInfo(features=self.config.features						)






					def        lowerCAmelCase_    (	self:							Optional[Any]						,					snake_case:							Union[str, Any]						)       ->     Optional[int]:
											if not self.config.data_files:
																	raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""						)
											snake_case_			:Optional[Any]						=      dl_manager.download_and_extract(self.config.data_files						)
											if isinstance(snake_case						,					(str, list, tuple)						):
																	snake_case_			:Union[str, Any]						=      data_files
																	if isinstance(snake_case						,					snake_case						):
																							snake_case_			:Optional[int]						=      [files]
																	snake_case_			:List[Any]						=      [dl_manager.iter_files(snake_case						) for file in files]
																	return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,					gen_kwargs={"""files""": files}						)]
											snake_case_			:List[str]						=      []
											for split_name, files in data_files.items():
																	if isinstance(snake_case						,					snake_case						):
																							snake_case_			:Dict						=      [files]
																	snake_case_			:Any						=      [dl_manager.iter_files(snake_case						) for file in files]
																	splits.append(datasets.SplitGenerator(name=snake_case						,					gen_kwargs={"""files""": files}						)						)
											return splits






					def        lowerCAmelCase_    (	self:							Tuple						,					snake_case:							pa.Table						)       ->     pa.Table:
											if self.config.features is not None:
																	snake_case_			:Union[str, Any]						=      self.config.features.arrow_schema
																	if all(not require_storage_cast(snake_case						) for feature in self.config.features.values()						):
																							# cheaper cast
																							snake_case_			:List[Any]						=      pa_table.cast(snake_case						)
																	else:
																							# more expensive cast; allows str <-> int/float or str to Audio for example
																							snake_case_			:List[Any]						=      table_cast(snake_case						,					snake_case						)
																	return pa_table
											else:
																	return pa_table.cast(pa.schema({"""text""": pa.string()}						)						)






					def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							str						)       ->     List[Any]:
											snake_case_			:int						=      list(self.config.features						) if self.config.features is not None else ["""text"""]
											for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case						)						):
																	# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
																	with open(snake_case						,					encoding=self.config.encoding						,					errors=self.config.errors						) as f:
																							if self.config.sample_by == "line":
																													snake_case_			:Any						=      0
																													while True:
																																			snake_case_			:Any						=      f.read(self.config.chunksize						)
																																			if not batch:
																																									break
																																			batch += f.readline()  # finish current line
																																			# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																			snake_case_			:Dict						=      StringIO(snake_case						).readlines()
																																			if not self.config.keep_linebreaks:
																																									snake_case_			:Optional[int]						=      [line.rstrip("""\n"""						) for line in batch]
																																			snake_case_			:Optional[Any]						=      pa.Table.from_arrays([pa.array(snake_case						)]						,					names=snake_case						)
																																			# Uncomment for debugging (will print the Arrow table size and elements)
																																			# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																			# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																			yield (file_idx, batch_idx), self._cast_table(snake_case						)
																																			batch_idx += 1
																							elif self.config.sample_by == "paragraph":
																													snake_case_			:Tuple						=      0
																													snake_case_			:Union[str, Any]						=      """"""
																													while True:
																																			snake_case_			:Dict						=      f.read(self.config.chunksize						)
																																			if not new_batch:
																																									break
																																			batch += new_batch
																																			batch += f.readline()  # finish current line
																																			snake_case_			:List[Any]						=      batch.split("""\n\n"""						)
																																			snake_case_			:str						=      pa.Table.from_arrays(
																																			    [pa.array([example for example in batch[:-1] if example]						)]						,					names=snake_case						)
																																			# Uncomment for debugging (will print the Arrow table size and elements)
																																			# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																			# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																			yield (file_idx, batch_idx), self._cast_table(snake_case						)
																																			batch_idx += 1
																																			snake_case_			:List[str]						=      batch[-1]
																													if batch:
																																			snake_case_			:str						=      pa.Table.from_arrays([pa.array([batch]						)]						,					names=snake_case						)
																																			yield (file_idx, batch_idx), self._cast_table(snake_case						)
																							elif self.config.sample_by == "document":
																													snake_case_			:Dict						=      f.read()
																													snake_case_			:Optional[int]						=      pa.Table.from_arrays([pa.array([text]						)]						,					names=snake_case						)
																													yield file_idx, self._cast_table(snake_case						)





"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


__a     =    {
    "configuration_xlm_prophetnet": ["XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLMProphetNetConfig"],
}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a     =    ["XLMProphetNetTokenizer"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a     =    [
						    "XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "XLMProphetNetDecoder",
						    "XLMProphetNetEncoder",
						    "XLMProphetNetForCausalLM",
						    "XLMProphetNetForConditionalGeneration",
						    "XLMProphetNetModel",
						    "XLMProphetNetPreTrainedModel",
						]


if TYPE_CHECKING:
						from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_xlm_prophetnet import (
												    XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
												    XLMProphetNetDecoder,
												    XLMProphetNetEncoder,
												    XLMProphetNetForCausalLM,
												    XLMProphetNetForConditionalGeneration,
												    XLMProphetNetModel,
												    XLMProphetNetPreTrainedModel,
												)

else:
						import sys

						__a     =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
