'''simple docstring'''




from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
		from .data.data_collator import DataCollator
		from .modeling_utils import PreTrainedModel
		from .tokenization_utils_base import PreTrainedTokenizerBase
		from .trainer_callback import TrainerCallback
		from .trainer_utils import EvalPrediction, PredictionOutput
		from .training_args import TrainingArguments


__SCREAMING_SNAKE_CASE	:					List[Any]					=  logging.get_logger(__name__)

class      lowerCamelCase_     (a_   ):
						'''simple docstring'''

						def __init__(	self						:    Any						, A						:    Tuple = None						, A						:    str = None						, A						:    Optional[int] = None						, A						:    str = None						, A						:    Tuple = None						, A						:    int = None						, A						:    Dict = None						, A						:    Tuple = None						, A						:    List[str] = None						, A						:    Tuple = (None, None)						, A						:    int = None						, ):
													super().__init__(
													    model=snake_case__						, args=snake_case__						, data_collator=snake_case__						, train_dataset=snake_case__						, eval_dataset=snake_case__						, tokenizer=snake_case__						, model_init=snake_case__						, compute_metrics=snake_case__						, callbacks=snake_case__						, optimizers=snake_case__						, preprocess_logits_for_metrics=snake_case__						, )

													# Override self.model.generation_config if a GenerationConfig is specified in args.
													# Priority: args.generation_config > model.generation_config > default GenerationConfig.
													if self.args.generation_config is not None:
																				_UpperCAmelCase					:  Dict											=   self.load_generation_config(self.args.generation_config	)
																				_UpperCAmelCase					:  Dict											=   gen_config




						@staticmethod
						def 					_A     (	A						:    int	):


													if isinstance(snake_case__						, snake_case__	):
																				return deepcopy(snake_case__	)

													# str or Path
													_UpperCAmelCase					:  Dict											=   Path(snake_case__	) if isinstance(snake_case__						, snake_case__	) else gen_config_arg
													_UpperCAmelCase					:  int											=   None

													# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
													# This step is required in order to determine config_file_name
													if pretrained_model_name.is_file():
																				_UpperCAmelCase					:  Dict											=   pretrained_model_name.name
																				_UpperCAmelCase					:  str											=   pretrained_model_name.parent
													# dir path
													elif pretrained_model_name.is_dir():
																				pass
													# model id or URL
													else:
																				_UpperCAmelCase					:  Dict											=   gen_config_arg

													_UpperCAmelCase					:  Tuple											=   GenerationConfig.from_pretrained(snake_case__						, snake_case__	)
													return gen_config




						def 					_A     (	self						:    Union[str, Any]						, A						:    List[str] = None						, A						:    List[Any] = None						, A						:    List[Any] = "eval"						, **A						:    Optional[Any]						, ):
													_UpperCAmelCase					:  Optional[int]											=   gen_kwargs.copy()
													if gen_kwargs.get("max_length"	) is None and gen_kwargs.get("max_new_tokens"	) is None:
																				_UpperCAmelCase					:  int											=   self.args.generation_max_length
													_UpperCAmelCase					:  Tuple											=   (
													    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
													)
													_UpperCAmelCase					:  Dict											=   gen_kwargs

													return super().evaluate(snake_case__						, ignore_keys=snake_case__						, metric_key_prefix=snake_case__	)




						def 					_A     (	self						:    Union[str, Any]						, A						:    Optional[int]						, A						:    List[Any] = None						, A						:    Optional[Any] = "test"						, **A						:    Dict						, ):
													_UpperCAmelCase					:  int											=   gen_kwargs.copy()
													if gen_kwargs.get("max_length"	) is None and gen_kwargs.get("max_new_tokens"	) is None:
																				_UpperCAmelCase					:  List[str]											=   self.args.generation_max_length
													_UpperCAmelCase					:  List[Any]											=   (
													    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
													)
													_UpperCAmelCase					:  Optional[Any]											=   gen_kwargs

													return super().predict(snake_case__						, ignore_keys=snake_case__						, metric_key_prefix=snake_case__	)




						def 					_A     (	self						:    Dict						, A						:    Tuple						, A						:    Union[str, Any]						, A						:    Optional[int]						, A						:    Any = None						, **A						:    Optional[Any]						, ):


													if not self.args.predict_with_generate or prediction_loss_only:
																				return super().prediction_step(
																				    snake_case__						, snake_case__						, prediction_loss_only=snake_case__						, ignore_keys=snake_case__	)

													_UpperCAmelCase					:  Union[str, Any]											=   "labels" in inputs
													_UpperCAmelCase					:  Any											=   self._prepare_inputs(snake_case__	)

													# XXX: adapt synced_gpus for fairscale as well
													# Priority (handled in generate):
													# gen_kwargs > model.generation_config > default GenerationConfig()

													if len(snake_case__	) == 0 and hasattr(self						, "_gen_kwargs"	):
																				_UpperCAmelCase					:  int											=   self._gen_kwargs.copy()

													if gen_kwargs.get("max_length"	) is None and gen_kwargs.get("max_new_tokens"	) is None:
																				_UpperCAmelCase					:  int											=   self.model.config.max_length
													_UpperCAmelCase					:  Any											=   (
													    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.model.config.num_beams
													)
													_UpperCAmelCase					:  Union[str, Any]											=   True if is_deepspeed_zeroa_enabled() else False
													_UpperCAmelCase					:  Optional[int]											=   (
													    gen_kwargs["synced_gpus"] if gen_kwargs.get("synced_gpus"	) is not None else default_synced_gpus
													)

													# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
													# (otherwise, it would continue generating from the padded `decoder_input_ids`)
													if (
													    "labels" in inputs
													    and "decoder_input_ids" in inputs
													    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
													):
																				_UpperCAmelCase					:  List[Any]											=   {k: v for k, v in inputs.items() if k != "decoder_input_ids"}
													_UpperCAmelCase					:  List[str]											=   self.model.generate(**snake_case__						, **snake_case__	)

													# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
													# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
													# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
													if self.model.generation_config._from_model_config:
																				_UpperCAmelCase					:  Dict											=   False

													# Retrieves GenerationConfig from model.generation_config
													_UpperCAmelCase					:  List[Any]											=   self.model.generation_config
													# in case the batch is shorter than max length, the output should be padded
													if generated_tokens.shape[-1] < gen_config.max_length:
																				_UpperCAmelCase					:  Dict											=   self._pad_tensors_to_max_len(snake_case__						, gen_config.max_length	)
													elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
																				_UpperCAmelCase					:  str											=   self._pad_tensors_to_max_len(snake_case__						, gen_config.max_new_tokens + 1	)

													with torch.no_grad():
																				if has_labels:
																											with self.compute_loss_context_manager():
																																		_UpperCAmelCase					:  Optional[Any]											=   model(**snake_case__	)
																											if self.label_smoother is not None:
																																		_UpperCAmelCase					:  List[Any]											=   self.label_smoother(snake_case__						, inputs["labels"]	).mean().detach()
																											else:
																																		_UpperCAmelCase					:  Optional[Any]											=   (outputs["loss"] if isinstance(snake_case__						, snake_case__	) else outputs[0]).mean().detach()
																				else:
																											_UpperCAmelCase					:  Optional[Any]											=   None

													if self.args.prediction_loss_only:
																				return loss, None, None

													if has_labels:
																				_UpperCAmelCase					:  Optional[Any]											=   inputs["labels"]
																				if labels.shape[-1] < gen_config.max_length:
																											_UpperCAmelCase					:  str											=   self._pad_tensors_to_max_len(snake_case__						, gen_config.max_length	)
																				elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																											_UpperCAmelCase					:  Union[str, Any]											=   self._pad_tensors_to_max_len(snake_case__						, gen_config.max_new_tokens + 1	)
													else:
																				_UpperCAmelCase					:  Tuple											=   None

													return loss, generated_tokens, labels





						def 					_A     (	self						:    str						, A						:    Tuple						, A						:    Tuple	):


													if self.tokenizer is not None and hasattr(self.tokenizer						, "pad_token_id"	):
																				# If PAD token is not defined at least EOS token has to be defined
																				_UpperCAmelCase					:  Any											=   (
																				    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
																				)
													else:
																				if self.model.config.pad_token_id is not None:
																											_UpperCAmelCase					:  List[str]											=   self.model.config.pad_token_id
																				else:
																											raise ValueError("Pad_token_id must be set in the configuration of the model, in order to pad tensors"	)

													_UpperCAmelCase					:  Dict											=   pad_token_id * torch.ones(
													    (tensor.shape[0], max_length)						, dtype=tensor.dtype						, device=tensor.device	)
													_UpperCAmelCase					:  List[Any]											=   tensor
													return padded_tensor


'''simple docstring'''




import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter

class      lowerCamelCase_     (TensorFormatter[Mapping, np.ndarray, Mapping]   ):
      '''simple docstring'''

      def __init__(	self						:    str						, A						:    List[str]=None						, **A						:    Optional[Any]	):
             super().__init__(features=A	)
             _UpperCAmelCase					:  Tuple											=   np_array_kwargs




      def 					_A     (	self						:    str						, A						:    Tuple	):
             if isinstance(A						, A	):
                    if column and all(
                        isinstance(A						, np.ndarray	) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column	):
                           return np.stack(A	)
                    else:
                           # don't use np.array(column, dtype=object)
                           # since it fails in certain cases
                           # see https://stackoverflow.com/q/51005699
                           _UpperCAmelCase					:  List[str]											=   np.empty(len(A	)						, dtype=A	)
                           _UpperCAmelCase					:  int											=   column
                           return out
             return column




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             if isinstance(A						, (str, bytes, type(A	))	):
                    return value
             elif isinstance(A						, (np.character, np.ndarray)	) and np.issubdtype(value.dtype						, np.character	):
                    return value
             elif isinstance(A						, np.number	):
                    return value

             _UpperCAmelCase					:  List[str]											=   {}

             if isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.integer	):
                    _UpperCAmelCase					:  str											=   {"dtype": np.intaa}
             elif isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.floating	):
                    _UpperCAmelCase					:  int											=   {"dtype": np.floataa}
             elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                    import PIL.Image

                    if isinstance(A						, PIL.Image.Image	):
                           return np.asarray(A						, **self.np_array_kwargs	)

             return np.asarray(A						, **{**default_dtype, **self.np_array_kwargs}	)




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             # support for torch, tf, jax etc.
             if config.TORCH_AVAILABLE and "torch" in sys.modules:
                    import torch

                    if isinstance(A						, torch.Tensor	):
                           return self._tensorize(data_struct.detach().cpu().numpy()[()]	)
             if hasattr(A						, "__array__"	) and not isinstance(A						, (np.ndarray, np.character, np.number)	):
                    _UpperCAmelCase					:  Dict											=   data_struct.__array__()
             # support for nested types like struct of list of struct
             if isinstance(A						, np.ndarray	):
                    if data_struct.dtype == object:
                           return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             if isinstance(A						, (list, tuple)	):
                    return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             return self._tensorize(A	)




      def 					_A     (	self						:    List[Any]						, A						:    dict	):
             return map_nested(self._recursive_tensorize						, A						, map_list=A	)




      def 					_A     (	self						:    List[str]						, A						:    pa.Table	):
             _UpperCAmelCase					:  str											=   self.numpy_arrow_extractor().extract_row(A	)
             _UpperCAmelCase					:  Dict											=   self.python_features_decoder.decode_row(A	)
             return self.recursive_tensorize(A	)




      def 					_A     (	self						:    str						, A						:    pa.Table	):
             _UpperCAmelCase					:  List[str]											=   self.numpy_arrow_extractor().extract_column(A	)
             _UpperCAmelCase					:  Optional[Any]											=   self.python_features_decoder.decode_column(A						, pa_table.column_names[0]	)
             _UpperCAmelCase					:  Tuple											=   self.recursive_tensorize(A	)
             _UpperCAmelCase					:  Tuple											=   self._consolidate(A	)
             return column





      def 					_A     (	self						:    Union[str, Any]						, A						:    pa.Table	):
             _UpperCAmelCase					:  Any											=   self.numpy_arrow_extractor().extract_batch(A	)
             _UpperCAmelCase					:  Any											=   self.python_features_decoder.decode_batch(A	)
             _UpperCAmelCase					:  List[str]											=   self.recursive_tensorize(A	)
             for column_name in batch:
                    _UpperCAmelCase					:  Optional[Any]											=   self._consolidate(batch[column_name]	)
             return batch

