



"""simple docstring"""
import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


__a     =    datasets.utils.logging.get_logger(__name__)
@dataclass
class     lowerCamelCase  (      datasets.BuilderConfig    ):






     '''simple docstring'''





     _A		:      Optional[datasets.Features]						=					None

class     lowerCamelCase  (      datasets.ArrowBasedBuilder    ):






     '''simple docstring'''





     _A		:      Optional[Any]						=					ArrowConfig






     def        lowerCAmelCase_    (	self:							int						)       ->     List[Any]:
           return datasets.DatasetInfo(features=self.config.features						)






     def        lowerCAmelCase_    (	self:							Dict						,					snake_case:							List[str]						)       ->     str:
           if not self.config.data_files:
                 raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""						)
           snake_case_			:Any						=      dl_manager.download_and_extract(self.config.data_files						)
           if isinstance(snake_case						,					(str, list, tuple)						):
                 snake_case_			:List[str]						=      data_files
                 if isinstance(snake_case						,					snake_case						):
                       snake_case_			:List[str]						=      [files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 snake_case_			:str						=      [dl_manager.iter_files(snake_case						) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,					gen_kwargs={"""files""": files}						)]
           snake_case_			:List[str]						=      []
           for split_name, files in data_files.items():
                 if isinstance(snake_case						,					snake_case						):
                       snake_case_			:str						=      [files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 snake_case_			:Optional[Any]						=      [dl_manager.iter_files(snake_case						) for file in files]
                 # Infer features is they are stoed in the arrow schema
                 if self.info.features is None:
                       for file in itertools.chain.from_iterable(snake_case						):
                             with open(snake_case						,					"""rb"""						) as f:
                                   snake_case_			:List[str]						=      datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case						).schema						)
                             break
                 splits.append(datasets.SplitGenerator(name=snake_case						,					gen_kwargs={"""files""": files}						)						)
           return splits






     def        lowerCAmelCase_    (	self:							Union[str, Any]						,					snake_case:							pa.Table						)       ->     pa.Table:
           if self.info.features is not None:
                 # more expensive cast to support nested features with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 snake_case_			:Tuple						=      table_cast(snake_case						,					self.info.features.arrow_schema						)
           return pa_table






     def        lowerCAmelCase_    (	self:							Optional[int]						,					snake_case:							Union[str, Any]						)       ->     List[Any]:
           for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case						)						):
                 with open(snake_case						,					"""rb"""						) as f:
                       try:
                             for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case						)						):
                                   snake_case_			:Union[str, Any]						=      pa.Table.from_batches([record_batch]						)
                                   # Uncomment for debugging (will print the Arrow table size and elements)
                                   # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                   # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                   yield f"""{file_idx}_{batch_idx}""", self._cast_table(snake_case						)
                       except ValueError as e:
                             logger.error(f"""Failed to read file '{file}' with error {type(snake_case						)}: {e}"""						)
                             raise





"""simple docstring"""
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch

import diffusers.utils.hub_utils
class     lowerCamelCase  (      unittest.TestCase    ):






					'''simple docstring'''





					@patch("""diffusers.utils.hub_utils.get_full_repo_name"""						)
					def        lowerCAmelCase_    (	self:							int						,					snake_case:							Mock						)       ->     None:
											snake_case_			:str						=      """full_repo_name"""
											with TemporaryDirectory() as tmpdir:
																	# Dummy args values
																	snake_case_			:int						=      Mock()
																	snake_case_			:Any						=      tmpdir
																	snake_case_			:int						=      0
																	snake_case_			:Optional[int]						=      """hub_token"""
																	snake_case_			:str						=      """dataset_name"""
																	snake_case_			:List[str]						=      0.0_1
																	snake_case_			:Dict						=      100_000
																	snake_case_			:Optional[Any]						=      10_000
																	snake_case_			:Optional[Any]						=      0.0_1
																	snake_case_			:Dict						=      0.0_2
																	snake_case_			:Dict						=      0.0_3
																	snake_case_			:Union[str, Any]						=      0.0_0_0_5
																	snake_case_			:Optional[Any]						=      0.0_0_0_0_0_1
																	snake_case_			:List[Any]						=      1
																	snake_case_			:Optional[Any]						=      10
																	snake_case_			:Dict						=      0.0_0_1
																	snake_case_			:Any						=      0.1
																	snake_case_			:List[Any]						=      0.2
																	snake_case_			:Any						=      True

																	# Model card mush be rendered and saved
																	diffusers.utils.hub_utils.create_model_card(snake_case						,					model_name="""model_name"""						)
																	self.assertTrue((Path(snake_case						) / """README.md""").is_file()						)
