
"""simple docstring"""




import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


UpperCAmelCase__				:   List[Any]   							=			get_logger(__name__)

def lowercase_  (       _snake_case       ,_snake_case       ,_snake_case       ,_snake_case       ,_snake_case       ,_snake_case = None       ,):
							# Model states
							for i, state in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	int									=       f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	List[Any]									=       os.path.join(_snake_case       ,_snake_case						)
														save(_snake_case       ,_snake_case						)
														logger.info(f'''Model weights saved in {output_model_file}'''						)
							# Optimizer states
							for i, opt in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	str									=       opt.state_dict()
														SCREAMING_SNAKE_CASE__						:	Optional[Any]									=       f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	str									=       os.path.join(_snake_case       ,_snake_case						)
														save(_snake_case       ,_snake_case						)
														logger.info(f'''Optimizer state saved in {output_optimizer_file}'''						)
							# Scheduler states
							for i, scheduler in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	int									=       scheduler.state_dict()
														SCREAMING_SNAKE_CASE__						:	Any									=       f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	Optional[Any]									=       os.path.join(_snake_case       ,_snake_case						)
														save(_snake_case       ,_snake_case						)
														logger.info(f'''Scheduler state saved in {output_scheduler_file}'''						)
							# GradScaler state
							if scaler is not None:
														SCREAMING_SNAKE_CASE__						:	str									=       scaler.state_dict()
														SCREAMING_SNAKE_CASE__						:	List[Any]									=       os.path.join(_snake_case       ,_snake_case						)
														torch.save(_snake_case       ,_snake_case						)
														logger.info(f'''Gradient scaler state saved in {output_scaler_file}'''						)
							# Random number generator states
							SCREAMING_SNAKE_CASE__						:	Optional[int]									=       {}
							SCREAMING_SNAKE_CASE__						:	int									=       f'''{RNG_STATE_NAME}_{process_index}.pkl'''
							SCREAMING_SNAKE_CASE__						:	Any									=       random.getstate()
							SCREAMING_SNAKE_CASE__						:	int									=       np.random.get_state()
							SCREAMING_SNAKE_CASE__						:	Optional[int]									=       torch.get_rng_state()
							if is_xpu_available():
														SCREAMING_SNAKE_CASE__						:	Union[str, Any]									=       torch.xpu.get_rng_state_all()
							else:
														SCREAMING_SNAKE_CASE__						:	str									=       torch.cuda.get_rng_state_all()
							if is_tpu_available():
														SCREAMING_SNAKE_CASE__						:	Tuple									=       xm.get_rng_state()
							SCREAMING_SNAKE_CASE__						:	Any									=       os.path.join(_snake_case       ,_snake_case						)
							torch.save(_snake_case       ,_snake_case						)
							logger.info(f'''Random states saved in {output_states_file}'''						)
							return output_dir

def lowercase_  (       _snake_case       ,_snake_case       ,_snake_case       ,_snake_case       ,_snake_case       ,_snake_case=None       ,_snake_case=None       ,**_snake_case       ,):
							if map_location not in [None, "cpu", "on_device"]:
														raise TypeError(
														    """Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"""						)
							if map_location is None:
														SCREAMING_SNAKE_CASE__						:	str									=       """cpu"""
							elif map_location == "on_device":
														SCREAMING_SNAKE_CASE__						:	Any									=       PartialState().device
							# Model states
							for i, model in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	Any									=       f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	Tuple									=       os.path.join(_snake_case       ,_snake_case						)
														models[i].load_state_dict(torch.load(_snake_case       ,map_location=_snake_case						)       ,**_snake_case						)
							logger.info("""All model weights loaded successfully"""						)

							# Optimizer states
							for i, opt in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	List[Any]									=       f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	int									=       os.path.join(_snake_case       ,_snake_case						)
														SCREAMING_SNAKE_CASE__						:	Optional[Any]									=       torch.load(_snake_case       ,map_location=_snake_case						)
														optimizers[i].load_state_dict(_snake_case						)
							logger.info("""All optimizer states loaded successfully"""						)

							# Scheduler states
							for i, scheduler in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:	Dict									=       f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
														SCREAMING_SNAKE_CASE__						:	str									=       os.path.join(_snake_case       ,_snake_case						)
														scheduler.load_state_dict(torch.load(_snake_case						)						)
							logger.info("""All scheduler states loaded successfully"""						)

							# GradScaler state
							if scaler is not None:
														SCREAMING_SNAKE_CASE__						:	Tuple									=       os.path.join(_snake_case       ,_snake_case						)
														scaler.load_state_dict(torch.load(_snake_case						)						)
														logger.info("""GradScaler state loaded successfully"""						)

							# Random states
							try:
														SCREAMING_SNAKE_CASE__						:	int									=       torch.load(os.path.join(_snake_case       ,f'''{RNG_STATE_NAME}_{process_index}.pkl'''						)						)
														random.setstate(states["""random_state"""]						)
														np.random.set_state(states["""numpy_random_seed"""]						)
														torch.set_rng_state(states["""torch_manual_seed"""]						)
														if is_xpu_available():
																					torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]						)
														else:
																					torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]						)
														if is_tpu_available():
																					xm.set_rng_state(states["""xm_seed"""]						)
														logger.info("""All random states loaded successfully"""						)
							except Exception:
														logger.info("""Could not load random states"""						)

def lowercase_  (       _snake_case       ,_snake_case       ,_snake_case = 0						):
							SCREAMING_SNAKE_CASE__						:	Tuple									=       Path(_snake_case						) / f'''custom_checkpoint_{index}.pkl'''
							logger.info(f'''Saving the state of {get_pretty_name(_snake_case						)} to {save_location}'''						)
							torch.save(obj.state_dict()       ,_snake_case						)

def lowercase_  (       _snake_case       ,_snake_case       ,_snake_case = 0						):
							SCREAMING_SNAKE_CASE__						:	Union[str, Any]									=       f'''{path}/custom_checkpoint_{index}.pkl'''
							logger.info(f'''Loading the state of {get_pretty_name(_snake_case						)} from {load_location}'''						)
							obj.load_state_dict(torch.load(_snake_case       ,map_location="""cpu"""						)						)






"""simple docstring"""




import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
					import sqlitea

					import sqlalchemy


UpperCAmelCase__				:   Union[str, Any]   							=			datasets.utils.logging.get_logger(__name__)



@dataclass
class      lowerCAmelCase_						(datasets.BuilderConfig  ):


				"""simple docstring"""



				__UpperCamelCase				:						Union[str, "sqlalchemy.sql.Selectable"] 	=       None
				__UpperCamelCase				:						Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] 	=       None
				__UpperCamelCase				:						Optional[Union[str, List[str]]] 	=       None
				__UpperCamelCase				:						bool 	=       True
				__UpperCamelCase				:						Optional[Union[List, Tuple, Dict]] 	=       None
				__UpperCamelCase				:						Optional[Union[List, Dict]] 	=       None
				__UpperCamelCase				:						Optional[List[str]] 	=       None
				__UpperCamelCase				:						Optional[int] 	=       10000
				__UpperCamelCase				:						Optional[datasets.Features] 	=       None



				def      __magic_name__   (self   ) ->		Dict:



											"""simple docstring"""




											if self.sql is None:
																		raise ValueError("""sql must be specified"""   )
											if self.con is None:
																		raise ValueError("""con must be specified"""   )



				def      __magic_name__   (self       ,	SCREAMING_SNAKE_CASE__       ,	SCREAMING_SNAKE_CASE__ = None       ,	) ->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:	List[Any]									=       config_kwargs.copy()
											# We need to stringify the Selectable object to make its hash deterministic

											# The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
											SCREAMING_SNAKE_CASE__						:	Dict									=       config_kwargs["""sql"""]
											if not isinstance(SCREAMING_SNAKE_CASE__       ,	SCREAMING_SNAKE_CASE__   ):
																		if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
																									import sqlalchemy

																									if isinstance(SCREAMING_SNAKE_CASE__       ,	sqlalchemy.sql.Selectable   ):
																																SCREAMING_SNAKE_CASE__						:	str									=       sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""   )[0] + """://"""   )
																																SCREAMING_SNAKE_CASE__						:	Tuple									=       str(sql.compile(dialect=engine.dialect   )   )
																																SCREAMING_SNAKE_CASE__						:	Any									=       sql_str
																									else:
																																raise TypeError(
																																    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(SCREAMING_SNAKE_CASE__   )}: {sql}'''   )
																		else:
																									raise TypeError(
																									    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(SCREAMING_SNAKE_CASE__   )}: {sql}'''   )
											SCREAMING_SNAKE_CASE__						:	Union[str, Any]									=       config_kwargs["""con"""]
											if not isinstance(SCREAMING_SNAKE_CASE__       ,	SCREAMING_SNAKE_CASE__   ):
																		SCREAMING_SNAKE_CASE__						:	Optional[Any]									=       id(SCREAMING_SNAKE_CASE__   )
																		logger.info(
																		    F'''SQL connection \'con\' of type {type(SCREAMING_SNAKE_CASE__   )} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''   )

											return super().create_config_id(SCREAMING_SNAKE_CASE__       ,	custom_features=SCREAMING_SNAKE_CASE__   )



				@property
				def      __magic_name__   (self   ) ->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:	Optional[int]									=       {
											    """index_col""": self.index_col,
											    """columns""": self.columns,
											    """params""": self.params,
											    """coerce_float""": self.coerce_float,
											    """parse_dates""": self.parse_dates,
											}
											return pd_read_sql_kwargs






class      lowerCAmelCase_						(datasets.ArrowBasedBuilder  ):


				"""simple docstring"""



				__UpperCamelCase				:						List[str] 	=       SqlConfig



				def      __magic_name__   (self   ) ->		List[Any]:



											"""simple docstring"""




											return datasets.DatasetInfo(features=self.config.features   )



				def      __magic_name__   (self       ,	SCREAMING_SNAKE_CASE__   ) ->		List[str]:



											"""simple docstring"""




											return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={}   )]



				def      __magic_name__   (self       ,	SCREAMING_SNAKE_CASE__   ) ->		pa.Table:



											"""simple docstring"""




											if self.config.features is not None:
																		SCREAMING_SNAKE_CASE__						:	List[str]									=       self.config.features.arrow_schema
																		if all(not require_storage_cast(SCREAMING_SNAKE_CASE__   ) for feature in self.config.features.values()   ):
																									# cheaper cast
																									SCREAMING_SNAKE_CASE__						:	Union[str, Any]									=       pa.Table.from_arrays([pa_table[field.name] for field in schema]       ,	schema=SCREAMING_SNAKE_CASE__   )
																		else:
																									# more expensive cast; allows str <-> int/float or str to Audio for example
																									SCREAMING_SNAKE_CASE__						:	Optional[int]									=       table_cast(SCREAMING_SNAKE_CASE__       ,	SCREAMING_SNAKE_CASE__   )
											return pa_table



				def      __magic_name__   (self   ) ->		Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:	List[Any]									=       self.config.chunksize
											SCREAMING_SNAKE_CASE__						:	Any									=       pd.read_sql(
											    self.config.sql       ,	self.config.con       ,	chunksize=SCREAMING_SNAKE_CASE__       ,	**self.config.pd_read_sql_kwargs   )
											SCREAMING_SNAKE_CASE__						:	str									=       [sql_reader] if chunksize is None else sql_reader
											for chunk_idx, df in enumerate(SCREAMING_SNAKE_CASE__   ):
																		SCREAMING_SNAKE_CASE__						:	int									=       pa.Table.from_pandas(SCREAMING_SNAKE_CASE__   )
																		yield chunk_idx, self._cast_table(SCREAMING_SNAKE_CASE__   )




