


'''simple docstring'''

import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCamelCase							:       int  						=     datasets.utils.logging.get_logger(__name__)
@dataclass
class   __lowerCAmelCase						(datasets.BuilderConfig					):





						'''simple docstring'''






						lowerCAmelCase__ :					Optional[datasets.Features]   =							None






class   __lowerCAmelCase						(datasets.ArrowBasedBuilder					):





						'''simple docstring'''






						lowerCAmelCase__ :					str   =							ArrowConfig



						def 				UpperCamelCase__		(self   :   Any			):

													'''simple docstring'''







													return datasets.DatasetInfo(features=self.config.features			)



						def 				UpperCamelCase__		(self   :   Union[str, Any]       ,						UpperCamelCase   :   Optional[Any]			):

													'''simple docstring'''







													if not self.config.data_files:
																				raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"			)
													lowercase__				      =			dl_manager.download_and_extract(self.config.data_files			)
													if isinstance(__A       ,						(str, list, tuple)			):
																				lowercase__				      =			data_files
																				if isinstance(__A       ,						__A			):
																											lowercase__				      =			[files]
																				# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																				lowercase__				      =			[dl_manager.iter_files(__A			) for file in files]
																				return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,						gen_kwargs={'''files''': files}			)]
													lowercase__				      =			[]
													for split_name, files in data_files.items():
																				if isinstance(__A       ,						__A			):
																											lowercase__				      =			[files]
																				# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																				lowercase__				      =			[dl_manager.iter_files(__A			) for file in files]
																				# Infer features is they are stoed in the arrow schema
																				if self.info.features is None:
																											for file in itertools.chain.from_iterable(__A			):
																																		with open(__A       ,						'''rb'''			) as f:
																																									lowercase__				      =			datasets.Features.from_arrow_schema(pa.ipc.open_stream(__A			).schema			)
																																		break
																				splits.append(datasets.SplitGenerator(name=__A       ,						gen_kwargs={'''files''': files}			)			)
													return splits



						def 				UpperCamelCase__		(self   :   Union[str, Any]       ,						UpperCamelCase   :   Dict			):

													'''simple docstring'''







													if self.info.features is not None:
																				# more expensive cast to support nested features with keys in a different order
																				# allows str <-> int/float or str to Audio for example
																				lowercase__				      =			table_cast(__A       ,						self.info.features.arrow_schema			)
													return pa_table






						def 				UpperCamelCase__		(self   :   List[Any]       ,						UpperCamelCase   :   Tuple			):

													'''simple docstring'''







													for file_idx, file in enumerate(itertools.chain.from_iterable(__A			)			):
																				with open(__A       ,						'''rb'''			) as f:
																											try:
																																		for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__A			)			):
																																									lowercase__				      =			pa.Table.from_batches([record_batch]			)
																																									# Uncomment for debugging (will print the Arrow table size and elements)
																																									# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																									# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																									yield f"{file_idx}_{batch_idx}", self._cast_table(__A			)
																											except ValueError as e:
																																		logger.error(f"Failed to read file '{file}' with error {type(__A			)}: {e}"			)
																																		raise








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase							:       str  						=     {'configuration_nat': ['NAT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NatConfig']}


try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:       Tuple  						=     [
     'NAT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'NatForImageClassification',
     'NatModel',
     'NatPreTrainedModel',
     'NatBackbone',
 ]

if TYPE_CHECKING:
 from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nat import (
      NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
      NatBackbone,
      NatForImageClassification,
      NatModel,
      NatPreTrainedModel,
  )

else:
 import sys

 lowerCamelCase							:       Optional[Any]  						=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




