




'''simple docstring'''







import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available





def 						snake_case_							(_a     :							str   ,       _a     :							List[str]   ,       _a     :							Optional[int]   ,       _a     :							Union[str, Any]=None					):
    UpperCAmelCase      	=		None
    # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
    if str(weight.dtype					) == "torch.bfloat16":
        # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
        UpperCAmelCase      	=		weight.view(torch.intaa					)
        UpperCAmelCase      	=		'''bfloat16'''
    UpperCAmelCase      	=		weight.cpu().numpy()
    UpperCAmelCase      	=		os.path.join(_a   ,       F"{weight_name}.dat"					)
    if index is not None:
        if dtype is None:
            UpperCAmelCase      	=		str(array.dtype					)
        UpperCAmelCase      	=		{'''dtype''': dtype, '''shape''': list(array.shape					)}
    if array.ndim == 0:
        UpperCAmelCase      	=		array[None]
    UpperCAmelCase      	=		np.memmap(_a   ,       dtype=array.dtype   ,       mode='''w+'''   ,       shape=array.shape					)
    UpperCAmelCase      	=		array[:]
    file_array.flush()
    return index





def 						snake_case_							(_a     :							Optional[int]   ,       _a     :							Optional[int]					):
    UpperCAmelCase      	=		tuple(weight_info['''shape''']					)
    if shape == ():
        # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
        UpperCAmelCase      	=		(1,)

    UpperCAmelCase      	=		weight_info['''dtype''']
    if dtype == "bfloat16":
        # NumPy does not support bfloat16 so this was saved as a int16
        UpperCAmelCase      	=		'''int16'''

    UpperCAmelCase      	=		np.memmap(_a   ,       dtype=_a   ,       shape=_a   ,       mode='''r'''					)

    if len(weight_info['''shape''']					) == 0:
        UpperCAmelCase      	=		weight[0]
    UpperCAmelCase      	=		torch.tensor(_a					)
    if weight_info["dtype"] == "bfloat16":
        UpperCAmelCase      	=		weight.view(torch.bfloataa					)

    return weight





def 						snake_case_							(_a     :							Union[str, Any]   ,       _a     :							Union[str, Any]					):
    if index is None or len(_a					) == 0:
        # Nothing to save
        return

    UpperCAmelCase      	=		os.path.join(_a   ,       '''index.json'''					)
    if os.path.isfile(_a					):
        with open(_a   ,       '''r'''   ,       encoding='''utf-8'''					) as f:
            UpperCAmelCase      	=		json.load(_a					)
    else:
        UpperCAmelCase      	=		{}
    current_index.update(_a					)

    with open(_a   ,       '''w'''   ,       encoding='''utf-8'''					) as f:
        json.dump(_a   ,       _a   ,       indent=2					)





def 						snake_case_							(_a     :							Union[str, os.PathLike]   ,       _a     :							Dict[str, torch.Tensor]					):
    os.makedirs(_a   ,       exist_ok=_a					)
    UpperCAmelCase      	=		{}
    for name, parameter in state_dict.items():
        UpperCAmelCase      	=		offload_weight(_a   ,       _a   ,       _a   ,       index=_a					)

    # Update index
    save_offload_index(_a   ,       _a					)


class       _a  (     __a			):
       def __init__(     self						:     List[str]       ,    lowercase						:     Mapping       ,    lowercase						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		dataset
           UpperCAmelCase      	=		prefix
       def __getitem__(     self						:     Tuple       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           return self.dataset[f"{self.prefix}{key}"]
       def __iter__(     self						:     Any    ):



           '''simple docstring'''

           return iter([key for key in self.dataset if key.startswith(self.prefix    )]    )







       def __len__(     self						:     Dict    ):



           '''simple docstring'''

           return len(self.dataset    )






class       _a  (     __a			):
       def __init__(     self						:     List[str]       ,    lowercase						:     Dict[str, torch.Tensor] = None       ,    lowercase						:     Optional[Union[str, os.PathLike]] = None       ,    lowercase						:     Mapping = None       ,    lowercase						:     Tuple=None       ,    ):



           '''simple docstring'''

           if state_dict is None and save_folder is None:
               raise ValueError('''Need either a `state_dict` or a `save_folder` containing offloaded weights.'''    )

           UpperCAmelCase      	=		{} if state_dict is None else state_dict
           UpperCAmelCase      	=		save_folder
           if index is None and save_folder is not None:
               with open(os.path.join(lowercase       ,    '''index.json'''    )    ) as f:
                   UpperCAmelCase      	=		json.load(lowercase    )
           UpperCAmelCase      	=		{} if index is None else index
           UpperCAmelCase      	=		list(self.state_dict.keys()    )
           self.all_keys.extend([key for key in self.index if key not in self.all_keys]    )
           UpperCAmelCase      	=		device
       def __getitem__(     self						:     Dict       ,    lowercase						:     str    ):



           '''simple docstring'''

           if key in self.state_dict:
               return self.state_dict[key]
           UpperCAmelCase      	=		self.index[key]
           if weight_info.get('''safetensors_file'''    ) is not None:
               if not is_safetensors_available():
                   raise ImportError('''These offloaded weights require the use of safetensors: `pip install safetensors`.'''    )

               from safetensors import safe_open

               UpperCAmelCase      	=		'''cpu''' if self.device is None else self.device
               with safe_open(weight_info['''safetensors_file''']       ,    framework='''pt'''       ,    device=lowercase    ) as f:
                   UpperCAmelCase      	=		f.get_tensor(weight_info.get('''weight_name'''       ,    lowercase    )    )

               if "dtype" in weight_info:
                   return tensor.to(getattr(lowercase       ,    weight_info['''dtype''']    )    )
               else:
                   return tensor

           UpperCAmelCase      	=		os.path.join(self.save_folder       ,    f"{key}.dat"    )
           return load_offloaded_weight(lowercase       ,    lowercase    )
       def __iter__(     self						:     Optional[Any]    ):



           '''simple docstring'''

           return iter(self.all_keys    )







       def __len__(     self						:     Optional[Any]    ):



           '''simple docstring'''

           return len(self.all_keys    )






def 						snake_case_							(_a     :							Dict[str, torch.Tensor]   ,       _a     :							List[str]					):
    UpperCAmelCase      	=		{}
    for module_name in submodule_names:
        # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
        # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
        result.update(
            {
                key: param
                for key, param in state_dict.items()
                if key == module_name or key.startswith(module_name + '''.'''					)
            }					)
    return result










'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class       _a  (     __a			):
       def __init__(     self						:     Any       ,    lowercase						:     NestedDataStructureLike[PathLike]       ,    lowercase						:     Optional[NamedSplit] = None       ,    lowercase						:     Optional[Features] = None       ,    lowercase						:     str = None       ,    lowercase						:     bool = False       ,    lowercase						:     bool = False       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     str       ,    ):



           '''simple docstring'''

           super().__init__(
               lowercase       ,    split=lowercase       ,    features=lowercase       ,    cache_dir=lowercase       ,    keep_in_memory=lowercase       ,    streaming=lowercase       ,    num_proc=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		path_or_paths if isinstance(lowercase       ,    lowercase    ) else {self.split: path_or_paths}
           UpperCAmelCase      	=		Csv(
               cache_dir=lowercase       ,    data_files=lowercase       ,    features=lowercase       ,    **lowercase       ,    )







       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           if self.streaming:
               UpperCAmelCase      	=		self.builder.as_streaming_dataset(split=self.split    )
           # Build regular (map-style) dataset
           else:
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None

               self.builder.download_and_prepare(
                   download_config=lowercase       ,    download_mode=lowercase       ,    verification_mode=lowercase       ,    base_path=lowercase       ,    num_proc=self.num_proc       ,    )
               UpperCAmelCase      	=		self.builder.as_dataset(
                   split=self.split       ,    verification_mode=lowercase       ,    in_memory=self.keep_in_memory    )
           return dataset






class       _a  :
       def __init__(     self						:     Optional[int]       ,    lowercase						:     Dataset       ,    lowercase						:     Union[PathLike, BinaryIO]       ,    lowercase						:     Optional[int] = None       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           if num_proc is not None and num_proc <= 0:
               raise ValueError(f"num_proc {num_proc} must be an integer > 0."    )

           UpperCAmelCase      	=		dataset
           UpperCAmelCase      	=		path_or_buf
           UpperCAmelCase      	=		batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
           UpperCAmelCase      	=		num_proc
           UpperCAmelCase      	=		'''utf-8'''
           UpperCAmelCase      	=		to_csv_kwargs
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''path_or_buf'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''header'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''index'''       ,    lowercase    )

           if isinstance(self.path_or_buf       ,    (str, bytes, os.PathLike)    ):
               with open(self.path_or_buf       ,    '''wb+'''    ) as buffer:
                   UpperCAmelCase      	=		self._write(file_obj=lowercase       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           else:
               UpperCAmelCase      	=		self._write(file_obj=self.path_or_buf       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           return written
       def    A				(     self						:     List[str]       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase      	=		args

           UpperCAmelCase      	=		query_table(
               table=self.dataset.data       ,    key=slice(lowercase       ,    offset + self.batch_size    )       ,    indices=self.dataset._indices       ,    )
           UpperCAmelCase      	=		batch.to_pandas().to_csv(
               path_or_buf=lowercase       ,    header=header if (offset == 0) else False       ,    index=lowercase       ,    **lowercase    )
           return csv_str.encode(self.encoding    )







       def    A				(     self						:     List[str]       ,    lowercase						:     BinaryIO       ,    lowercase						:     int       ,    lowercase						:     List[str]       ,    **lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		0

           if self.num_proc is None or self.num_proc == 1:
               for offset in logging.tqdm(
                   range(0       ,    len(self.dataset    )       ,    self.batch_size    )       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                   UpperCAmelCase      	=		self._batch_csv((offset, header, index, to_csv_kwargs)    )
                   written += file_obj.write(lowercase    )

           else:
               UpperCAmelCase	,					UpperCAmelCase      	=		len(self.dataset    ), self.batch_size
               with multiprocessing.Pool(self.num_proc    ) as pool:
                   for csv_str in logging.tqdm(
                       pool.imap(
                           self._batch_csv       ,    [(offset, header, index, to_csv_kwargs) for offset in range(0       ,    lowercase       ,    lowercase    )]       ,    )       ,    total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                       written += file_obj.write(lowercase    )

           return written




