


'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class  lowercase_					(   _a       ):


				def __init__(					self  , a  , a = None  , a = None  , a = None  , a = False  , a = False  , a = None  , **a  , ):
										super().__init__(
										    SCREAMING_SNAKE_CASE__  , split=SCREAMING_SNAKE_CASE__  , features=SCREAMING_SNAKE_CASE__  , cache_dir=SCREAMING_SNAKE_CASE__  , keep_in_memory=SCREAMING_SNAKE_CASE__  , streaming=SCREAMING_SNAKE_CASE__  , num_proc=SCREAMING_SNAKE_CASE__  , **SCREAMING_SNAKE_CASE__  , )
										UpperCamelCase__  =    path_or_paths if isinstance(SCREAMING_SNAKE_CASE__  , SCREAMING_SNAKE_CASE__   ) else {self.split: path_or_paths}
										UpperCamelCase__  =    Csv(
										    cache_dir=SCREAMING_SNAKE_CASE__  , data_files=SCREAMING_SNAKE_CASE__  , features=SCREAMING_SNAKE_CASE__  , **SCREAMING_SNAKE_CASE__  , )


				def        __a							(					self   ):
										# Build iterable dataset
										if self.streaming:
																UpperCamelCase__  =    self.builder.as_streaming_dataset(split=self.split   )
										# Build regular (map-style) dataset
										else:
																UpperCamelCase__  =    None
																UpperCamelCase__  =    None
																UpperCamelCase__  =    None
																UpperCamelCase__  =    None

																self.builder.download_and_prepare(
																    download_config=SCREAMING_SNAKE_CASE__  , download_mode=SCREAMING_SNAKE_CASE__  , verification_mode=SCREAMING_SNAKE_CASE__  , base_path=SCREAMING_SNAKE_CASE__  , num_proc=self.num_proc  , )
																UpperCamelCase__  =    self.builder.as_dataset(
																    split=self.split  , verification_mode=SCREAMING_SNAKE_CASE__  , in_memory=self.keep_in_memory   )
										return dataset

class  lowercase_					:


				def __init__(					self  , a  , a  , a = None  , a = None  , **a  , ):
										if num_proc is not None and num_proc <= 0:
																raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''   )

										UpperCamelCase__  =    dataset
										UpperCamelCase__  =    path_or_buf
										UpperCamelCase__  =    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
										UpperCamelCase__  =    num_proc
										UpperCamelCase__  =    "utf-8"
										UpperCamelCase__  =    to_csv_kwargs


				def        __a							(					self   ):
										UpperCamelCase__  =    self.to_csv_kwargs.pop("path_or_buf"  , SCREAMING_SNAKE_CASE__   )
										UpperCamelCase__  =    self.to_csv_kwargs.pop("header"  , SCREAMING_SNAKE_CASE__   )
										UpperCamelCase__  =    self.to_csv_kwargs.pop("index"  , SCREAMING_SNAKE_CASE__   )

										if isinstance(self.path_or_buf  , (str, bytes, os.PathLike)   ):
																with open(self.path_or_buf  , "wb+"   ) as buffer:
																						UpperCamelCase__  =    self._write(file_obj=SCREAMING_SNAKE_CASE__  , header=SCREAMING_SNAKE_CASE__  , index=SCREAMING_SNAKE_CASE__  , **self.to_csv_kwargs   )
										else:
																UpperCamelCase__  =    self._write(file_obj=self.path_or_buf  , header=SCREAMING_SNAKE_CASE__  , index=SCREAMING_SNAKE_CASE__  , **self.to_csv_kwargs   )
										return written


				def        __a							(					self  , a   ):
										UpperCamelCase__  =    args

										UpperCamelCase__  =    query_table(
										    table=self.dataset.data  , key=slice(SCREAMING_SNAKE_CASE__  , offset + self.batch_size   )  , indices=self.dataset._indices  , )
										UpperCamelCase__  =    batch.to_pandas().to_csv(
										    path_or_buf=SCREAMING_SNAKE_CASE__  , header=header if (offset == 0) else False  , index=SCREAMING_SNAKE_CASE__  , **SCREAMING_SNAKE_CASE__   )
										return csv_str.encode(self.encoding   )


				def        __a							(					self  , a  , a  , a  , **a   ):
										UpperCamelCase__  =    0

										if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0  , len(self.dataset   )  , self.batch_size   )  , unit="ba"  , disable=not logging.is_progress_bar_enabled()  , desc="Creating CSV from Arrow format"  , ):
																						UpperCamelCase__  =    self._batch_csv((offset, header, index, to_csv_kwargs)   )
																						written += file_obj.write(SCREAMING_SNAKE_CASE__   )

										else:
																UpperCamelCase__  =    len(self.dataset   ), self.batch_size
																with multiprocessing.Pool(self.num_proc   ) as pool:
																						for csv_str in logging.tqdm(
																						    pool.imap(
																						        self._batch_csv  , [(offset, header, index, to_csv_kwargs) for offset in range(0  , SCREAMING_SNAKE_CASE__  , SCREAMING_SNAKE_CASE__   )]  , )  , total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size  , unit="ba"  , disable=not logging.is_progress_bar_enabled()  , desc="Creating CSV from Arrow format"  , ):
																												written += file_obj.write(SCREAMING_SNAKE_CASE__   )

										return written









'''simple docstring'''







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


a__  : Any				=	{
    'configuration_tvlt': ['TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TvltConfig'],
    'feature_extraction_tvlt': ['TvltFeatureExtractor'],
    'processing_tvlt': ['TvltProcessor'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a__  : int				=	[
      'TVLT_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TvltModel',
      'TvltForPreTraining',
      'TvltForAudioVisualClassification',
      'TvltPreTrainedModel',
  ]

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a__  : List[Any]				=	['TvltImageProcessor']


if TYPE_CHECKING:
  from .configuration_tvlt import TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP, TvltConfig
  from .processing_tvlt import TvltProcessor
  from .feature_extraction_tvlt import TvltFeatureExtractor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tvlt import (
        TVLT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TvltForAudioVisualClassification,
        TvltForPreTraining,
        TvltModel,
        TvltPreTrainedModel,
    )

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .image_processing_tvlt import TvltImageProcessor


else:
  import sys

  a__  : Any				=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





