import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase     						=			datasets.utils.logging.get_logger(__name__)




@dataclass
class 				__lowerCAmelCase					(    datasets.BuilderConfig  ):



 '''simple docstring'''
 _lowerCamelCase  :		Optional[datasets.Features]			      =	None
 _lowerCamelCase  :		str			      =	"utf-8"
 _lowerCamelCase  :		Optional[str]			      =	None
 _lowerCamelCase  :		int			      =	10 << 20  # 10MB
 _lowerCamelCase  :		bool			      =	False
 _lowerCamelCase  :		str			      =	"line"







class 				__lowerCAmelCase					(    datasets.ArrowBasedBuilder  ):



 '''simple docstring'''
 _lowerCamelCase  :		Union[str, Any]			      =	TextConfig

 def    __snake_case   (     self    :						Tuple					)	->							Dict:
        return datasets.DatasetInfo(features=self.config.features					)

 def    __snake_case   (     self    :						List[Any]    ,     lowerCamelCase    :						Optional[int]					)	->							Dict:
        if not self.config.data_files:
               raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"					)
        _UpperCamelCase       :    Dict					  =					dl_manager.download_and_extract(self.config.data_files					)
        if isinstance(lowerCamelCase    ,     (str, list, tuple)					):
               _UpperCamelCase       :    Tuple					  =					data_files
               if isinstance(lowerCamelCase    ,     lowerCamelCase					):
                      _UpperCamelCase       :    Any					  =					[files]
               _UpperCamelCase       :    Dict					  =					[dl_manager.iter_files(lowerCamelCase					) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,     gen_kwargs={'files': files}					)]
        _UpperCamelCase       :    Dict					  =					[]
        for split_name, files in data_files.items():
               if isinstance(lowerCamelCase    ,     lowerCamelCase					):
                      _UpperCamelCase       :    Optional[int]					  =					[files]
               _UpperCamelCase       :    Tuple					  =					[dl_manager.iter_files(lowerCamelCase					) for file in files]
               splits.append(datasets.SplitGenerator(name=lowerCamelCase    ,     gen_kwargs={'files': files}					)					)
        return splits

 def    __snake_case   (     self    :						str    ,     lowerCamelCase    :						pa.Table					)	->							pa.Table:
        if self.config.features is not None:
               _UpperCamelCase       :    List[Any]					  =					self.config.features.arrow_schema
               if all(not require_storage_cast(lowerCamelCase					) for feature in self.config.features.values()					):
                      # cheaper cast
                      _UpperCamelCase       :    Optional[int]					  =					pa_table.cast(lowerCamelCase					)
               else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      _UpperCamelCase       :    Any					  =					table_cast(lowerCamelCase    ,     lowerCamelCase					)
               return pa_table
        else:
               return pa_table.cast(pa.schema({'text': pa.string()}					)					)

 def    __snake_case   (     self    :						Dict    ,     lowerCamelCase    :						Optional[int]					)	->							Dict:
        _UpperCamelCase       :    Optional[int]					  =					list(self.config.features					) if self.config.features is not None else ['text']
        for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase					)					):
               # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
               with open(lowerCamelCase    ,     encoding=self.config.encoding    ,     errors=self.config.errors					) as f:
                      if self.config.sample_by == "line":
                             _UpperCamelCase       :    Any					  =					0
                             while True:
                                    _UpperCamelCase       :    Any					  =					f.read(self.config.chunksize					)
                                    if not batch:
                                           break
                                    batch += f.readline()  # finish current line
                                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                    _UpperCamelCase       :    Tuple					  =					StringIO(lowerCamelCase					).readlines()
                                    if not self.config.keep_linebreaks:
                                           _UpperCamelCase       :    int					  =					[line.rstrip('\n'					) for line in batch]
                                    _UpperCamelCase       :    Optional[Any]					  =					pa.Table.from_arrays([pa.array(lowerCamelCase					)]    ,     names=lowerCamelCase					)
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(lowerCamelCase					)
                                    batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                             _UpperCamelCase       :    int					  =					0
                             _UpperCamelCase       :    Optional[Any]					  =					''
                             while True:
                                    _UpperCamelCase       :    Optional[int]					  =					f.read(self.config.chunksize					)
                                    if not new_batch:
                                           break
                                    batch += new_batch
                                    batch += f.readline()  # finish current line
                                    _UpperCamelCase       :    Tuple					  =					batch.split('\n\n'					)
                                    _UpperCamelCase       :    Optional[Any]					  =					pa.Table.from_arrays(
                                        [pa.array([example for example in batch[:-1] if example]					)]    ,     names=lowerCamelCase					)
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(lowerCamelCase					)
                                    batch_idx += 1
                                    _UpperCamelCase       :    List[Any]					  =					batch[-1]
                             if batch:
                                    _UpperCamelCase       :    int					  =					pa.Table.from_arrays([pa.array([batch]					)]    ,     names=lowerCamelCase					)
                                    yield (file_idx, batch_idx), self._cast_table(lowerCamelCase					)
                      elif self.config.sample_by == "document":
                             _UpperCamelCase       :    Optional[int]					  =					f.read()
                             _UpperCamelCase       :    Dict					  =					pa.Table.from_arrays([pa.array([text]					)]    ,     names=lowerCamelCase					)
                             yield file_idx, self._cast_table(lowerCamelCase					)




from copy import deepcopy
from unittest.case import TestCase

import pytest

from datasets.arrow_dataset import Dataset
from datasets.features import Audio, ClassLabel, Features, Image, Sequence, Value
from datasets.info import DatasetInfo
from datasets.tasks import (
    AudioClassification,
    AutomaticSpeechRecognition,
    ImageClassification,
    LanguageModeling,
    QuestionAnsweringExtractive,
    Summarization,
    TextClassification,
    task_template_from_dict,
)
from datasets.utils.py_utils import asdict


UpperCAmelCase     						=			{
    '''id''': '''5733be284776f41900661182''',
    '''title''': '''University_of_Notre_Dame''',
    '''context''': '''Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.''',
    '''question''': '''To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?''',
    '''answers''': {'''text''': ['''Saint Bernadette Soubirous'''], '''answer_start''': [5_15]},
}


@pytest.mark.parametrize(
    'task_cls'						,  [
        AudioClassification,
        AutomaticSpeechRecognition,
        ImageClassification,
        LanguageModeling,
        QuestionAnsweringExtractive,
        Summarization,
        TextClassification,
    ]						,  )
def SCREAMING_SNAKE_CASE			(       lowercase_:		Optional[int]			):
       _UpperCamelCase       :    int					  =					task_cls()
       _UpperCamelCase       :    List[Any]					  =					asdict(lowercase_			)
       _UpperCamelCase       :    Union[str, Any]					  =					task_template_from_dict(lowercase_			)
       assert task == reloaded




class 				__lowerCAmelCase					:



 '''simple docstring'''

 def    __snake_case   (     self    :						Dict					)	->							Any:
        _UpperCamelCase       :    List[Any]					  =					LanguageModeling(text_column='input_text'					)
        assert {"input_text": "text"} == task.column_mapping

 def    __snake_case   (     self    :						List[Any]					)	->							List[Any]:
        _UpperCamelCase       :    List[str]					  =					Features({'text': Value('string'					)}					)
        _UpperCamelCase       :    Optional[int]					  =					{'text_column': 'input_text'}
        _UpperCamelCase       :    Optional[Any]					  =					LanguageModeling.from_dict(lowerCamelCase					)
        assert "language-modeling" == task.task
        assert input_schema == task.input_schema







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						List[Any]					)	->							Union[str, Any]:
        _UpperCamelCase       :    Union[str, Any]					  =					sorted(['pos', 'neg']					)

 def    __snake_case   (     self    :						List[str]					)	->							Dict:
        _UpperCamelCase       :    List[str]					  =					TextClassification(text_column='input_text'    ,     label_column='input_label'					)
        self.assertDictEqual({'input_text': 'text', 'input_label': 'labels'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						Optional[Any]					)	->							Dict:
        _UpperCamelCase       :    Optional[Any]					  =					Features({'text': Value('string'					)}					)
        # Labels are cast to tuple during `TextClassification.__post_init__`, so we do the same here
        _UpperCamelCase       :    List[Any]					  =					Features({'labels': ClassLabel}					)
        _UpperCamelCase       :    Optional[int]					  =					{'text_column': 'input_text', 'label_column': 'input_labels'}
        _UpperCamelCase       :    str					  =					TextClassification.from_dict(lowerCamelCase					)
        self.assertEqual('text-classification'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)

 def    __snake_case   (     self    :						str					)	->							Optional[int]:
        _UpperCamelCase       :    Any					  =					TextClassification(text_column='input_text'    ,     label_column='input_label'					)
        self.assertEqual(task.label_schema['labels']    ,     lowerCamelCase					)
        _UpperCamelCase       :    str					  =					task.align_with_features(Features({'input_label': ClassLabel(names=self.labels					)}					)					)
        self.assertEqual(task.label_schema['labels']    ,     ClassLabel(names=self.labels					)					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						Dict					)	->							str:
        _UpperCamelCase       :    Dict					  =					QuestionAnsweringExtractive(
            context_column='input_context'    ,     question_column='input_question'    ,     answers_column='input_answers'					)
        self.assertDictEqual(
            {'input_context': 'context', 'input_question': 'question', 'input_answers': 'answers'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						Union[str, Any]					)	->							Tuple:
        _UpperCamelCase       :    Optional[int]					  =					Features({'question': Value('string'					), 'context': Value('string'					)}					)
        _UpperCamelCase       :    Any					  =					Features(
            {
                'answers': Sequence(
                    {
                        'text': Value('string'					),
                        'answer_start': Value('int32'					),
                    }					)
            }					)
        _UpperCamelCase       :    str					  =					{
            'context_column': 'input_input_context',
            'question_column': 'input_question',
            'answers_column': 'input_answers',
        }
        _UpperCamelCase       :    Optional[Any]					  =					QuestionAnsweringExtractive.from_dict(lowerCamelCase					)
        self.assertEqual('question-answering-extractive'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						Optional[Any]					)	->							List[str]:
        _UpperCamelCase       :    List[str]					  =					Summarization(text_column='input_text'    ,     summary_column='input_summary'					)
        self.assertDictEqual({'input_text': 'text', 'input_summary': 'summary'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						int					)	->							Dict:
        _UpperCamelCase       :    Tuple					  =					Features({'text': Value('string'					)}					)
        _UpperCamelCase       :    List[Any]					  =					Features({'summary': Value('string'					)}					)
        _UpperCamelCase       :    Any					  =					{'text_column': 'input_text', 'summary_column': 'input_summary'}
        _UpperCamelCase       :    Any					  =					Summarization.from_dict(lowerCamelCase					)
        self.assertEqual('summarization'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						Any					)	->							Optional[int]:
        _UpperCamelCase       :    Dict					  =					AutomaticSpeechRecognition(audio_column='input_audio'    ,     transcription_column='input_transcription'					)
        self.assertDictEqual({'input_audio': 'audio', 'input_transcription': 'transcription'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						List[Any]					)	->							int:
        _UpperCamelCase       :    Union[str, Any]					  =					Features({'audio': Audio()}					)
        _UpperCamelCase       :    Any					  =					Features({'transcription': Value('string'					)}					)
        _UpperCamelCase       :    Optional[int]					  =					{
            'audio_column': 'input_audio',
            'transcription_column': 'input_transcription',
        }
        _UpperCamelCase       :    int					  =					AutomaticSpeechRecognition.from_dict(lowerCamelCase					)
        self.assertEqual('automatic-speech-recognition'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						Union[str, Any]					)	->							Dict:
        _UpperCamelCase       :    Dict					  =					sorted(['pos', 'neg']					)

 def    __snake_case   (     self    :						Optional[int]					)	->							Optional[int]:
        _UpperCamelCase       :    Union[str, Any]					  =					AudioClassification(audio_column='input_audio'    ,     label_column='input_label'					)
        self.assertDictEqual({'input_audio': 'audio', 'input_label': 'labels'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						Tuple					)	->							Dict:
        _UpperCamelCase       :    List[str]					  =					Features({'audio': Audio()}					)
        _UpperCamelCase       :    Union[str, Any]					  =					Features({'labels': ClassLabel}					)
        _UpperCamelCase       :    Union[str, Any]					  =					{
            'audio_column': 'input_image',
            'label_column': 'input_label',
        }
        _UpperCamelCase       :    Tuple					  =					AudioClassification.from_dict(lowerCamelCase					)
        self.assertEqual('audio-classification'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)

 def    __snake_case   (     self    :						Any					)	->							List[Any]:
        _UpperCamelCase       :    Any					  =					AudioClassification(audio_column='input_audio'    ,     label_column='input_label'					)
        self.assertEqual(task.label_schema['labels']    ,     lowerCamelCase					)
        _UpperCamelCase       :    Union[str, Any]					  =					task.align_with_features(Features({'input_label': ClassLabel(names=self.labels					)}					)					)
        self.assertEqual(task.label_schema['labels']    ,     ClassLabel(names=self.labels					)					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						int					)	->							str:
        _UpperCamelCase       :    Tuple					  =					sorted(['pos', 'neg']					)

 def    __snake_case   (     self    :						Optional[Any]					)	->							Any:
        _UpperCamelCase       :    str					  =					ImageClassification(image_column='input_image'    ,     label_column='input_label'					)
        self.assertDictEqual({'input_image': 'image', 'input_label': 'labels'}    ,     task.column_mapping					)

 def    __snake_case   (     self    :						int					)	->							Dict:
        _UpperCamelCase       :    Dict					  =					Features({'image': Image()}					)
        _UpperCamelCase       :    Tuple					  =					Features({'labels': ClassLabel}					)
        _UpperCamelCase       :    Optional[int]					  =					{
            'image_column': 'input_image',
            'label_column': 'input_label',
        }
        _UpperCamelCase       :    List[str]					  =					ImageClassification.from_dict(lowerCamelCase					)
        self.assertEqual('image-classification'    ,     task.task					)
        self.assertEqual(lowerCamelCase    ,     task.input_schema					)
        self.assertEqual(lowerCamelCase    ,     task.label_schema					)

 def    __snake_case   (     self    :						List[str]					)	->							Union[str, Any]:
        _UpperCamelCase       :    Any					  =					ImageClassification(image_column='input_image'    ,     label_column='input_label'					)
        self.assertEqual(task.label_schema['labels']    ,     lowerCamelCase					)
        _UpperCamelCase       :    Optional[Any]					  =					task.align_with_features(Features({'input_label': ClassLabel(names=self.labels					)}					)					)
        self.assertEqual(task.label_schema['labels']    ,     ClassLabel(names=self.labels					)					)







class 				__lowerCAmelCase					(    a_  ):



 '''simple docstring'''

 def    __snake_case   (     self    :						List[str]					)	->							Dict:
        _UpperCamelCase       :    List[str]					  =					DatasetInfo(task_templates=QuestionAnsweringExtractive()					)
        _UpperCamelCase       :    Tuple					  =					Dataset.from_dict({k: [v] for k, v in SAMPLE_QUESTION_ANSWERING_EXTRACTIVE.items()}    ,     info=lowerCamelCase					)
        assert isinstance(dataset.info.task_templates    ,     lowerCamelCase					)
        assert len(dataset.info.task_templates					) == 1

        def keep_task(lowerCamelCase    :						Dict					):
               return x

        def dont_keep_task(lowerCamelCase    :						List[Any]					):
               _UpperCamelCase       :    Union[str, Any]					  =					deepcopy(lowerCamelCase					)
               _UpperCamelCase       :    List[str]					  =					0
               return out

        _UpperCamelCase       :    Any					  =					dataset.map(lowerCamelCase					)
        assert mapped_dataset.info.task_templates == dataset.info.task_templates
        # reload from cache
        _UpperCamelCase       :    Any					  =					dataset.map(lowerCamelCase					)
        assert mapped_dataset.info.task_templates == dataset.info.task_templates

        _UpperCamelCase       :    int					  =					dataset.map(lowerCamelCase					)
        assert mapped_dataset.info.task_templates == []
        # reload from cache
        _UpperCamelCase       :    int					  =					dataset.map(lowerCamelCase					)
        assert mapped_dataset.info.task_templates == []

 def    __snake_case   (     self    :						Union[str, Any]					)	->							Any:
        _UpperCamelCase       :    List[str]					  =					Features({'text': Value('string'					), 'label': ClassLabel(names=('pos', 'neg')					)}					)
        _UpperCamelCase       :    List[str]					  =					TextClassification(text_column='text'    ,     label_column='label'					)
        _UpperCamelCase       :    List[str]					  =					DatasetInfo(features=lowerCamelCase    ,     task_templates=lowerCamelCase					)
        _UpperCamelCase       :    Any					  =					Dataset.from_dict({'text': ['A sentence.'], 'label': ['pos']}    ,     info=lowerCamelCase					)

        def process(lowerCamelCase    :						Any					):
               return example

        _UpperCamelCase       :    Dict					  =					dataset.remove_columns('label'					)
        _UpperCamelCase       :    Union[str, Any]					  =					modified_dataset.map(lowerCamelCase					)
        assert mapped_dataset.info.task_templates == []



