




import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		)
class 		UpperCAmelCase       :







						'''simple docstring'''





						def __init__(						self							:					Optional[Any]	,     __lowercase							:					Optional[int]	,     __lowercase							:					Tuple=13	,     __lowercase							:					Any=7	,     __lowercase							:					List[Any]=True	,     __lowercase							:					Optional[int]=True	,     __lowercase							:					List[str]=True	,     __lowercase							:					Dict=True	,     __lowercase							:					Dict=99	,     __lowercase							:					int=[1, 1, 2]	,     __lowercase							:					Optional[int]=1	,     __lowercase							:					Optional[int]=32	,     __lowercase							:					List[str]=4	,     __lowercase							:					List[Any]=8	,     __lowercase							:					List[Any]=37	,     __lowercase							:					Any="gelu_new"	,     __lowercase							:					Optional[Any]=0.1	,     __lowercase							:					Tuple=0.1	,     __lowercase							:					Optional[int]=0.0	,     __lowercase							:					Any=5_12	,     __lowercase							:					Union[str, Any]=3	,     __lowercase							:					Tuple=0.02	,     __lowercase							:					Union[str, Any]=3	,     __lowercase							:					Optional[int]=4	,     __lowercase							:					Union[str, Any]=None	,     __lowercase							:					List[str]=False	,     ):






							"""simple docstring"""






							snake_case_		   =					parent
							snake_case_		   =					batch_size
							snake_case_		   =					seq_length
							snake_case_		   =					is_training
							snake_case_		   =					use_input_mask
							snake_case_		   =					use_token_type_ids
							snake_case_		   =					use_labels
							snake_case_		   =					vocab_size
							snake_case_		   =					block_sizes
							snake_case_		   =					num_decoder_layers
							snake_case_		   =					d_model
							snake_case_		   =					n_head
							snake_case_		   =					d_head
							snake_case_		   =					d_inner
							snake_case_		   =					hidden_act
							snake_case_		   =					hidden_dropout
							snake_case_		   =					attention_dropout
							snake_case_		   =					activation_dropout
							snake_case_		   =					max_position_embeddings
							snake_case_		   =					type_vocab_size
							snake_case_		   =					2
							snake_case_		   =					num_labels
							snake_case_		   =					num_choices
							snake_case_		   =					scope
							snake_case_		   =					initializer_std

							# Used in the tests to check the size of the first attention layer
							snake_case_		   =					n_head
							# Used in the tests to check the size of the first hidden state
							snake_case_		   =					self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							snake_case_		   =					sum(self.block_sizes						) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
								snake_case_		   =					self.num_hidden_layers + 2






						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.vocab_size						)

							snake_case_		   =					None
							if self.use_input_mask:
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     vocab_size=2						)

							snake_case_		   =					None
							if self.use_token_type_ids:
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.type_vocab_size						)

							snake_case_		   =					None
							snake_case_		   =					None
							snake_case_		   =					None
							if self.use_labels:
								snake_case_		   =					ids_tensor([self.batch_size]	,     self.type_sequence_label_size						)
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     self.num_labels						)
								snake_case_		   =					ids_tensor([self.batch_size]	,     self.num_choices						)
								snake_case_		   =					ids_tensor([self.batch_size, self.seq_length]	,     1						)

							snake_case_		   =					self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    fake_token_labels,
							)






						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							return FunnelConfig(
							    vocab_size=self.vocab_size	,     block_sizes=self.block_sizes	,     num_decoder_layers=self.num_decoder_layers	,     d_model=self.d_model	,     n_head=self.n_head	,     d_head=self.d_head	,     d_inner=self.d_inner	,     hidden_act=self.hidden_act	,     hidden_dropout=self.hidden_dropout	,     attention_dropout=self.attention_dropout	,     activation_dropout=self.activation_dropout	,     max_position_embeddings=self.max_position_embeddings	,     type_vocab_size=self.type_vocab_size	,     initializer_std=self.initializer_std	,     )






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					List[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					List[str]	,     __lowercase							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					str	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, self.seq_length, self.d_model)						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Any	,     __lowercase							:					int	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					str	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelBaseModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 2, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 3, self.d_model)						)

							snake_case_		   =					False
							snake_case_		   =					model(__lowercase						)
							self.parent.assertEqual(result.last_hidden_state.shape	,     (self.batch_size, 2, self.d_model)						)






						def 	snake_case__						(						self							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     __lowercase							:					str	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForPreTraining(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length)						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					str	,     __lowercase							:					List[str]	,     __lowercase							:					Any	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					int	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelForMaskedLM(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length, self.vocab_size)						)






						def 	snake_case__						(						self							:					Any	,     __lowercase							:					Dict	,     __lowercase							:					List[Any]	,     __lowercase							:					Any	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     __lowercase							:					Any	,     __lowercase							:					List[str]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForSequenceClassification(__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.num_labels)						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					str	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     __lowercase							:					Dict	,     __lowercase							:					List[str]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_choices
							snake_case_		   =					FunnelForMultipleChoice(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					input_ids.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					token_type_ids.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					input_mask.unsqueeze(1						).expand(-1	,     self.num_choices	,     -1						).contiguous()
							snake_case_		   =					model(
							    __lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase	,     )
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.num_choices)						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[int]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					int	,     __lowercase							:					Dict	,     __lowercase							:					List[str]	,     __lowercase							:					Tuple	,     __lowercase							:					List[str]	,     __lowercase							:					List[Any]	,     ):






							"""simple docstring"""






							snake_case_		   =					self.num_labels
							snake_case_		   =					FunnelForTokenClassification(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape	,     (self.batch_size, self.seq_length, self.num_labels)						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					Any	,     __lowercase							:					List[Any]	,     __lowercase							:					List[Any]	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Dict	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[int]	,     __lowercase							:					Optional[int]	,     ):






							"""simple docstring"""






							snake_case_		   =					FunnelForQuestionAnswering(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		   =					model(
							    __lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase	,     start_positions=__lowercase	,     end_positions=__lowercase	,     )
							self.parent.assertEqual(result.start_logits.shape	,     (self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape	,     (self.batch_size, self.seq_length)						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				(
							    snake_case_
							)   ,				
							)		   =					config_and_inputs
							snake_case_		   =					{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class 		UpperCAmelCase       (    UpperCAmelCase__	,       UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCAmelCase_							=       (
						    {
						        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
						        '''fill-mask''': FunnelForMaskedLM,
						        '''question-answering''': FunnelForQuestionAnswering,
						        '''text-classification''': FunnelForSequenceClassification,
						        '''token-classification''': FunnelForTokenClassification,
						        '''zero-shot''': FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)






						def 	snake_case__						(						self							:					Dict	,     __lowercase							:					Any	,     __lowercase							:					Optional[Any]	,     __lowercase							:					Optional[int]=False						):






							"""simple docstring"""






							snake_case_		   =					super()._prepare_for_class(__lowercase	,     __lowercase	,     return_labels=__lowercase						)

							if return_labels:
								if model_class in get_values(__lowercase						):
									snake_case_		   =					torch.zeros(
									    (self.model_tester.batch_size, self.model_tester.seq_length)	,     dtype=torch.long	,     device=__lowercase						)
							return inputs_dict






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					FunnelModelTester(self						)
							snake_case_		   =					ConfigTester(self	,     config_class=__lowercase						)






						def 	snake_case__						(						self							:					Optional[int]						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*__lowercase						)






						def 	snake_case__						(						self							:					Tuple						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__lowercase						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowercase						)






						def 	snake_case__						(						self							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowercase						)






						def 	snake_case__						(						self							:					List[Any]	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							if hasattr(__lowercase	,     "weight"						) and module.weight is not None:
								module.weight.data.fill_(3						)
							if hasattr(__lowercase	,     "bias"						) and module.bias is not None:
								module.bias.data.fill_(3						)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__lowercase	,     __lowercase						) and getattr(__lowercase	,     __lowercase						) is not None:
									snake_case_		   =					getattr(__lowercase	,     __lowercase						)
									weight.data.fill_(3						)







@require_torch
class 		UpperCAmelCase       (    UpperCAmelCase__	,       unittest.TestCase      ):







						'''simple docstring'''





						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       False
						lowerCAmelCase_							=       (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					FunnelModelTester(self	,     base=__lowercase						)
							snake_case_		   =					ConfigTester(self	,     config_class=__lowercase						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def 	snake_case__						(						self							:					Optional[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*__lowercase						)






						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__lowercase						)






						def 	snake_case__						(						self							:					int						):






							"""simple docstring"""






							snake_case_		   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__lowercase						)






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_   ,				snake_case_		   =					self.model_tester.prepare_config_and_inputs_for_common()
							snake_case_		   =					True

							for model_class in self.all_model_classes:
								if model_class.__name__ == "FunnelBaseModel":
									continue
								snake_case_		   =					model_class(__lowercase						)
								model.to(__lowercase						)
								model.train()
								snake_case_		   =					self._prepare_for_class(__lowercase	,     __lowercase	,     return_labels=__lowercase						)
								snake_case_		   =					model(**__lowercase						).loss
								loss.backward()






						def 	snake_case__						(						self							:					str	,     __lowercase							:					Optional[Any]						):






							"""simple docstring"""






							if hasattr(__lowercase	,     "weight"						) and module.weight is not None:
								module.weight.data.fill_(3						)
							if hasattr(__lowercase	,     "bias"						) and module.bias is not None:
								module.bias.data.fill_(3						)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__lowercase	,     __lowercase						) and getattr(__lowercase	,     __lowercase						) is not None:
									snake_case_		   =					getattr(__lowercase	,     __lowercase						)
									weight.data.fill_(3						)







@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase       (    unittest.TestCase      ):







						'''simple docstring'''





						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					13
							snake_case_		   =					7
							snake_case_		   =					torch.arange(0	,     batch_size * sequence_length						).long().reshape(__lowercase	,     __lowercase						)
							snake_case_		   =					[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
							snake_case_		   =					torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]						)

							snake_case_		   =					FunnelModel.from_pretrained("sgugger/funnel-random-tiny"						)
							snake_case_		   =					model(__lowercase	,     token_type_ids=__lowercase						)[0].abs()

							snake_case_		   =					torch.tensor(2344.8352						)
							snake_case_		   =					torch.tensor(0.8052						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)

							snake_case_		   =					torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]						)
							snake_case_		   =					model(__lowercase	,     attention_mask=__lowercase	,     token_type_ids=__lowercase						)[0].abs()

							snake_case_		   =					torch.tensor(2343.8425						)
							snake_case_		   =					torch.tensor(0.8049						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)






						@slow
						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							snake_case_		   =					FunnelTokenizer.from_pretrained("huggingface/funnel-small"						)
							snake_case_		   =					FunnelModel.from_pretrained("huggingface/funnel-small"						)
							snake_case_		   =					tokenizer("Hello! I am the Funnel Transformer model."	,     return_tensors="pt"						)
							snake_case_		   =					model(**__lowercase						)[0]

							snake_case_		   =					torch.tensor(235.7246						)
							snake_case_		   =					torch.tensor(0.0256						)
							self.assertTrue(torch.allclose(output.sum()	,     __lowercase	,     atol=1E-4						)						)
							self.assertTrue(torch.allclose(output.mean()	,     __lowercase	,     atol=1E-4						)						)






import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowercase__    :					str					=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 		UpperCAmelCase       (    datasets.BuilderConfig      ):







						'''simple docstring'''





						lowerCAmelCase_							=       None







class 		UpperCAmelCase       (    datasets.ArrowBasedBuilder      ):







						'''simple docstring'''





						lowerCAmelCase_							=       ArrowConfig






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							return datasets.DatasetInfo(features=self.config.features						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Union[str, Any]						):






							"""simple docstring"""






							if not self.config.data_files:
								raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"						)
							snake_case_		   =					dl_manager.download_and_extract(self.config.data_files						)
							if isinstance(__lowercase	,     (str, list, tuple)						):
								snake_case_		   =					data_files
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,     gen_kwargs={"files": files}						)]
							snake_case_		   =					[]
							for split_name, files in data_files.items():
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								# Infer features is they are stoed in the arrow schema
								if self.info.features is None:
									for file in itertools.chain.from_iterable(__lowercase						):
										with open(__lowercase	,     "rb"						) as f:
											snake_case_		   =					datasets.Features.from_arrow_schema(pa.ipc.open_stream(__lowercase						).schema						)
										break
								splits.append(datasets.SplitGenerator(name=__lowercase	,     gen_kwargs={"files": files}						)						)
							return splits






						def 	snake_case__						(						self							:					str	,     __lowercase							:					pa.Table						):






							"""simple docstring"""






							if self.info.features is not None:
								# more expensive cast to support nested features with keys in a different order
								# allows str <-> int/float or str to Audio for example
								snake_case_		   =					table_cast(__lowercase	,     self.info.features.arrow_schema						)
							return pa_table






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[Any]						):






							"""simple docstring"""






							for file_idx, file in enumerate(itertools.chain.from_iterable(__lowercase						)						):
								with open(__lowercase	,     "rb"						) as f:
									try:
										for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__lowercase						)						):
											snake_case_		   =					pa.Table.from_batches([record_batch]						)
											# Uncomment for debugging (will print the Arrow table size and elements)
											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield f"{file_idx}_{batch_idx}", self._cast_table(__lowercase						)
									except ValueError as e:
										logger.error(f"Failed to read file '{file}' with error {type(__lowercase						)}: {e}"						)
										raise
