





'''simple docstring'''
import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


a__							:							List[Any]			=datasets.utils.logging.get_logger(__name__)

@dataclass
class 					snake_case					(   datasets.BuilderConfig	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : Optional[datasets.Features]      		=None
 SCREAMING_SNAKE_CASE_     : str      		="utf-8"
 SCREAMING_SNAKE_CASE_     : Optional[str]      		=None
 SCREAMING_SNAKE_CASE_     : int      		=10 << 20  # 10MB
 SCREAMING_SNAKE_CASE_     : bool      		=False
 SCREAMING_SNAKE_CASE_     : str      		="line"






class 					snake_case					(   datasets.ArrowBasedBuilder	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : Dict      		=TextConfig






 def 	_lowerCamelCase (	self	:							int     ):
        return datasets.DatasetInfo(features=self.config.features     )






 def 	_lowerCamelCase (	self	:							Union[str, Any]							,     __A	:							Optional[Any]     ):
        if not self.config.data_files:
               raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''     )
        __UpperCamelCase	=  dl_manager.download_and_extract(self.config.data_files     )
        if isinstance(__A							,     (str, list, tuple)     ):
               __UpperCamelCase	=  data_files
               if isinstance(__A							,     __A     ):
                      __UpperCamelCase	=  [files]
               __UpperCamelCase	=  [dl_manager.iter_files(__A     ) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,     gen_kwargs={'files': files}     )]
        __UpperCamelCase	=  []
        for split_name, files in data_files.items():
               if isinstance(__A							,     __A     ):
                      __UpperCamelCase	=  [files]
               __UpperCamelCase	=  [dl_manager.iter_files(__A     ) for file in files]
               splits.append(datasets.SplitGenerator(name=__A							,     gen_kwargs={'files': files}     )     )
        return splits






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							pa.Table     ):
        if self.config.features is not None:
               __UpperCamelCase	=  self.config.features.arrow_schema
               if all(not require_storage_cast(__A     ) for feature in self.config.features.values()     ):
                      # cheaper cast
                      __UpperCamelCase	=  pa_table.cast(__A     )
               else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __UpperCamelCase	=  table_cast(__A							,     __A     )
               return pa_table
        else:
               return pa_table.cast(pa.schema({'text': pa.string()}     )     )






 def 	_lowerCamelCase (	self	:							int							,     __A	:							Tuple     ):
        __UpperCamelCase	=  list(self.config.features     ) if self.config.features is not None else ['text']
        for file_idx, file in enumerate(itertools.chain.from_iterable(__A     )     ):
               # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
               with open(__A							,     encoding=self.config.encoding							,     errors=self.config.errors     ) as f:
                      if self.config.sample_by == "line":
                             __UpperCamelCase	=  0
                             while True:
                                    __UpperCamelCase	=  f.read(self.config.chunksize     )
                                    if not batch:
                                           break
                                    batch += f.readline()  # finish current line
                                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                    __UpperCamelCase	=  StringIO(__A     ).readlines()
                                    if not self.config.keep_linebreaks:
                                           __UpperCamelCase	=  [line.rstrip('\n'     ) for line in batch]
                                    __UpperCamelCase	=  pa.Table.from_arrays([pa.array(__A     )]							,     names=__A     )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(__A     )
                                    batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                             __UpperCamelCase	=  0
                             __UpperCamelCase	=  ''
                             while True:
                                    __UpperCamelCase	=  f.read(self.config.chunksize     )
                                    if not new_batch:
                                           break
                                    batch += new_batch
                                    batch += f.readline()  # finish current line
                                    __UpperCamelCase	=  batch.split('\n\n'     )
                                    __UpperCamelCase	=  pa.Table.from_arrays(
                                        [pa.array([example for example in batch[:-1] if example]     )]							,     names=__A     )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(__A     )
                                    batch_idx += 1
                                    __UpperCamelCase	=  batch[-1]
                             if batch:
                                    __UpperCamelCase	=  pa.Table.from_arrays([pa.array([batch]     )]							,     names=__A     )
                                    yield (file_idx, batch_idx), self._cast_table(__A     )
                      elif self.config.sample_by == "document":
                             __UpperCamelCase	=  f.read()
                             __UpperCamelCase	=  pa.Table.from_arrays([pa.array([text]     )]							,     names=__A     )
                             yield file_idx, self._cast_table(__A     )







'''simple docstring'''
a__							:							int			='''
# Transformers installation
! pip install transformers datasets evaluate
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

a__							:							List[Any]			=[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
a__							:							Dict			={
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}
