



import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


__lowerCamelCase								=   datasets.utils.logging.get_logger(__name__)



@dataclass
class 						_snake_case (     datasets.BuilderConfig			):





	"""simple docstring"""



	lowerCamelCase_       =     None







class 						_snake_case (     datasets.ArrowBasedBuilder			):





	"""simple docstring"""



	lowerCamelCase_       =     ArrowConfig
	def lowercase_						(  self   )		->							Union[str, Any]:




			"""simple docstring"""







			return datasets.DatasetInfo(features=self.config.features   )
	def lowercase_						(  self		,      a   )		->							int:




			"""simple docstring"""







			if not self.config.data_files:
					raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''   )
			_A          =      dl_manager.download_and_extract(self.config.data_files   )
			if isinstance(snake_case__		,      (str, list, tuple)   ):
					_A          =      data_files
					if isinstance(snake_case__		,      snake_case__   ):
							_A          =      [files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					_A          =      [dl_manager.iter_files(snake_case__   ) for file in files]
					return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,      gen_kwargs={'''files''': files}   )]
			_A          =      []
			for split_name, files in data_files.items():
					if isinstance(snake_case__		,      snake_case__   ):
							_A          =      [files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					_A          =      [dl_manager.iter_files(snake_case__   ) for file in files]
					# Infer features is they are stoed in the arrow schema
					if self.info.features is None:
							for file in itertools.chain.from_iterable(snake_case__   ):
									with open(snake_case__		,      '''rb'''   ) as f:
											_A          =      datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case__   ).schema   )
									break
					splits.append(datasets.SplitGenerator(name=snake_case__		,      gen_kwargs={'''files''': files}   )   )
			return splits
	def lowercase_						(  self		,      a   )		->							int:




			"""simple docstring"""







			if self.info.features is not None:
					# more expensive cast to support nested features with keys in a different order
					# allows str <-> int/float or str to Audio for example
					_A          =      table_cast(snake_case__		,      self.info.features.arrow_schema   )
			return pa_table



	def lowercase_						(  self		,      a   )		->							List[Any]:




			"""simple docstring"""







			for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__   )   ):
					with open(snake_case__		,      '''rb'''   ) as f:
							try:
									for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case__   )   ):
											_A          =      pa.Table.from_batches([record_batch]   )
											# Uncomment for debugging (will print the Arrow table size and elements)
											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield f'''{file_idx}_{batch_idx}''', self._cast_table(snake_case__   )
							except ValueError as e:
									logger.error(f'''Failed to read file \'{file}\' with error {type(snake_case__   )}: {e}'''   )
									raise




import warnings

from .generation import FlaxGenerationMixin



class 						_snake_case (     lowerCamelCase			):





 """simple docstring"""



 warnings.warn(
     '''Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will '''
     '''be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead.'''   ,lowerCamelCase   ,)