





import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


__A    :   Any          =       datasets.utils.logging.get_logger(__name__)

@dataclass
class      A_	(datasets.BuilderConfig      ):
	UpperCAmelCase__ =      None

class      A_	(datasets.ArrowBasedBuilder      ):
	UpperCAmelCase__ =      ArrowConfig
	def 					_lowercase    (   self		):






				'''simple docstring'''







				return datasets.DatasetInfo(features=self.config.features		)
	def 					_lowercase    (   self							, _A		):






				'''simple docstring'''







				if not self.config.data_files:
							raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""		)
				UpperCAmelCase       		=    dl_manager.download_and_extract(self.config.data_files		)
				if isinstance(_A							, (str, list, tuple)		):
							UpperCAmelCase       		=    data_files
							if isinstance(_A							, _A		):
										UpperCAmelCase       		=    [files]
							# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
							UpperCAmelCase       		=    [dl_manager.iter_files(_A		) for file in files]
							return [datasets.SplitGenerator(name=datasets.Split.TRAIN							, gen_kwargs={'''files''': files}		)]
				UpperCAmelCase       		=    []
				for split_name, files in data_files.items():
							if isinstance(_A							, _A		):
										UpperCAmelCase       		=    [files]
							# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
							UpperCAmelCase       		=    [dl_manager.iter_files(_A		) for file in files]
							# Infer features is they are stoed in the arrow schema
							if self.info.features is None:
										for file in itertools.chain.from_iterable(_A		):
													with open(_A							, '''rb'''		) as f:
																UpperCAmelCase       		=    datasets.Features.from_arrow_schema(pa.ipc.open_stream(_A		).schema		)
													break
							splits.append(datasets.SplitGenerator(name=_A							, gen_kwargs={'''files''': files}		)		)
				return splits
	def 					_lowercase    (   self							, _A		):






				'''simple docstring'''







				if self.info.features is not None:
							# more expensive cast to support nested features with keys in a different order
							# allows str <-> int/float or str to Audio for example
							UpperCAmelCase       		=    table_cast(_A							, self.info.features.arrow_schema		)
				return pa_table







	def 					_lowercase    (   self							, _A		):






				'''simple docstring'''







				for file_idx, file in enumerate(itertools.chain.from_iterable(_A		)		):
							with open(_A							, '''rb'''		) as f:
										try:
													for batch_idx, record_batch in enumerate(pa.ipc.open_stream(_A		)		):
																UpperCAmelCase       		=    pa.Table.from_batches([record_batch]		)
																# Uncomment for debugging (will print the Arrow table size and elements)
																# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																yield F"""{file_idx}_{batch_idx}""", self._cast_table(_A		)
										except ValueError as e:
													logger.error(F"""Failed to read file '{file}' with error {type(_A		)}: {e}"""		)
													raise










import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    str:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file.jsonl'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    Union[str, Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_utf16_encoded.jsonl'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''			)
			with open(UpperCamelCase__      ,      '''w'''      ,      encoding='''utf-16'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    Optional[Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_with_list_of_dicts.json'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        [
            {"col_1": -1},
            {"col_1": 1, "col_2": 2},
            {"col_1": 10, "col_2": 20}
        ]
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    int:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_with_list_of_dicts_field.json'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {
            "field1": 1,
            "field2": "aabb",
            "field3": [
                {"col_1": -1},
                {"col_1": 1, "col_2": 2},
                {"col_1": 10, "col_2": 20}
            ]
        }
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''      ,      [
        ('''jsonl_file''', {}),
        ('''jsonl_file_utf16_encoded''', {'''encoding''': '''utf-16'''}),
        ('''json_file_with_list_of_dicts''', {}),
        ('''json_file_with_list_of_dicts_field''', {'''field''': '''field3'''}),
    ]      ,      )
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__      ,      UpperCamelCase__      ,      UpperCamelCase__			)  ->    Dict:







			'''simple docstring'''

			UpperCAmelCase       		=    Json(**UpperCamelCase__			)
			UpperCAmelCase       		=    json._generate_tables([[request.getfixturevalue(UpperCamelCase__			)]]			)
			UpperCAmelCase       		=    pa.concat_tables([table for _, table in generator]			)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}






@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''      ,      [
        (
            '''jsonl_file''',
            {'''features''': Features({'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			)},
        ),
        (
            '''json_file_with_list_of_dicts''',
            {'''features''': Features({'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			)},
        ),
        (
            '''json_file_with_list_of_dicts_field''',
            {
                '''field''': '''field3''',
                '''features''': Features(
                    {'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			),
            },
        ),
    ]      ,      )
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__      ,      UpperCamelCase__      ,      UpperCamelCase__			)  ->    List[Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    Json(**UpperCamelCase__			)
			UpperCAmelCase       		=    json._generate_tables([[request.getfixturevalue(UpperCamelCase__			)]]			)
			UpperCAmelCase       		=    pa.concat_tables([table for _, table in generator]			)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}



