






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    ElectraTokenizer
							_UpperCAmelCase	      =    ElectraTokenizerFast
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    filter_non_english
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											super().setUp()

											snake_case__													=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case__													=       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: List[str]  )						->						Union[str, Any]:
											snake_case__													=       'UNwant\u00E9d,running'
											snake_case__													=       'unwanted, running'
											return input_text, output_text
							def lowerCAmelCase_	(       self: List[str]  )						->						List[Any]:
											snake_case__													=       self.tokenizer_class(self.vocab_file  )

											snake_case__													=       tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(UpperCamelCase    ,		['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase  )    ,		[9, 6, 7, 12, 10, 11]  )
							def lowerCAmelCase_	(       self: str  )						->						Tuple:
											if not self.test_rust_tokenizer:
															return

											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											# With lower casing
											snake_case__													=       self.get_tokenizer(do_lower_case=UpperCamelCase  )
											snake_case__													=       self.get_rust_tokenizer(do_lower_case=UpperCamelCase  )

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											snake_case__													=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,		['ah', '\u535A', '\u63A8', 'zz']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[Any]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['h\u00E9llo']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: str  )						->						List[str]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
							def lowerCAmelCase_	(       self: Any  )						->						Dict:
											snake_case__													=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											snake_case__													=       {}
											for i, token in enumerate(UpperCamelCase  ):
															snake_case__													=       i
											snake_case__													=       WordpieceTokenizer(vocab=UpperCamelCase    ,		unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )    ,		[]  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,		['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,		['[UNK]', 'runn', '##ing']  )
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Tuple:
											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
							def lowerCAmelCase_	(       self: str  )						->						Any:
											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )
							@slow
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											snake_case__													=       tokenizer.encode('sequence builders'    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       tokenizer.encode('multi-sequence build'    ,		add_special_tokens=UpperCamelCase  )

											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase  )
											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase    ,		UpperCamelCase  )

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			snake_case__													=       tokenizer_r.encode_plus(
																			    UpperCamelCase    ,		return_attention_mask=UpperCamelCase    ,		return_token_type_ids=UpperCamelCase    ,		return_offsets_mapping=UpperCamelCase    ,		add_special_tokens=UpperCamelCase    ,		)

																			snake_case__													=       tokenizer_r.do_lower_case if hasattr(UpperCamelCase    ,		'do_lower_case'  ) else False
																			snake_case__													=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																			self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping']  )


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       ['的', '人', '有']
											snake_case__													=       ''.join(UpperCamelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       True
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

																			snake_case__													=       False
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case__													=       [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase  )
																			]
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )












from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
							from .data.data_collator import DataCollator
							from .modeling_utils import PreTrainedModel
							from .tokenization_utils_base import PreTrainedTokenizerBase
							from .trainer_callback import TrainerCallback
							from .trainer_utils import EvalPrediction, PredictionOutput
							from .training_args import TrainingArguments


__UpperCamelCase   :						Tuple							=  logging.get_logger(__name__)


class 					__SCREAMING_SNAKE_CASE(		a_		):
							def __init__(       self: Dict    ,		UpperCamelCase: Union["PreTrainedModel", nn.Module] = None    ,		UpperCamelCase: "TrainingArguments" = None    ,		UpperCamelCase: Optional["DataCollator"] = None    ,		UpperCamelCase: Optional[Dataset] = None    ,		UpperCamelCase: Optional[Union[Dataset, Dict[str, Dataset]]] = None    ,		UpperCamelCase: Optional["PreTrainedTokenizerBase"] = None    ,		UpperCamelCase: Optional[Callable[[], "PreTrainedModel"]] = None    ,		UpperCamelCase: Optional[Callable[["EvalPrediction"], Dict]] = None    ,		UpperCamelCase: Optional[List["TrainerCallback"]] = None    ,		UpperCamelCase: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None)    ,		UpperCamelCase: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None    ,		)						->						Tuple:
											super().__init__(
											    model=UpperCamelCase    ,		args=UpperCamelCase    ,		data_collator=UpperCamelCase    ,		train_dataset=UpperCamelCase    ,		eval_dataset=UpperCamelCase    ,		tokenizer=UpperCamelCase    ,		model_init=UpperCamelCase    ,		compute_metrics=UpperCamelCase    ,		callbacks=UpperCamelCase    ,		optimizers=UpperCamelCase    ,		preprocess_logits_for_metrics=UpperCamelCase    ,		)

											# Override self.model.generation_config if a GenerationConfig is specified in args.
											# Priority: args.generation_config > model.generation_config > default GenerationConfig.
											if self.args.generation_config is not None:
															snake_case__													=       self.load_generation_config(self.args.generation_config  )
															snake_case__													=       gen_config
							@staticmethod
							def lowerCAmelCase_	(       UpperCamelCase: Union[str, GenerationConfig]  )						->						GenerationConfig:

											# GenerationConfig provided, nothing to do
											if isinstance(UpperCamelCase    ,		UpperCamelCase  ):
															return deepcopy(UpperCamelCase  )

											# str or Path
											snake_case__													=       Path(UpperCamelCase  ) if isinstance(UpperCamelCase    ,		UpperCamelCase  ) else gen_config_arg
											snake_case__													=       None

											# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
											# This step is required in order to determine config_file_name
											if pretrained_model_name.is_file():
															snake_case__													=       pretrained_model_name.name
															snake_case__													=       pretrained_model_name.parent
											# dir path
											elif pretrained_model_name.is_dir():
															pass
											# model id or URL
											else:
															snake_case__													=       gen_config_arg

											snake_case__													=       GenerationConfig.from_pretrained(UpperCamelCase    ,		UpperCamelCase  )
											return gen_config
							def lowerCAmelCase_	(       self: Optional[int]    ,		UpperCamelCase: Optional[Dataset] = None    ,		UpperCamelCase: Optional[List[str]] = None    ,		UpperCamelCase: str = "eval"    ,		**UpperCamelCase: Dict    ,		)						->						Dict[str, float]:
											snake_case__													=       gen_kwargs.copy()
											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.args.generation_max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.args.generation_num_beams
											)
											snake_case__													=       gen_kwargs

											return super().evaluate(UpperCamelCase    ,		ignore_keys=UpperCamelCase    ,		metric_key_prefix=UpperCamelCase  )
							def lowerCAmelCase_	(       self: Tuple    ,		UpperCamelCase: Dataset    ,		UpperCamelCase: Optional[List[str]] = None    ,		UpperCamelCase: str = "test"    ,		**UpperCamelCase: Dict    ,		)						->						"PredictionOutput":
											snake_case__													=       gen_kwargs.copy()
											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.args.generation_max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.args.generation_num_beams
											)
											snake_case__													=       gen_kwargs

											return super().predict(UpperCamelCase    ,		ignore_keys=UpperCamelCase    ,		metric_key_prefix=UpperCamelCase  )
							def lowerCAmelCase_	(       self: Tuple    ,		UpperCamelCase: nn.Module    ,		UpperCamelCase: Dict[str, Union[torch.Tensor, Any]]    ,		UpperCamelCase: bool    ,		UpperCamelCase: Optional[List[str]] = None    ,		**UpperCamelCase: List[Any]    ,		)						->						Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:

											if not self.args.predict_with_generate or prediction_loss_only:
															return super().prediction_step(
															    UpperCamelCase    ,		UpperCamelCase    ,		prediction_loss_only=UpperCamelCase    ,		ignore_keys=UpperCamelCase  )

											snake_case__													=       'labels' in inputs
											snake_case__													=       self._prepare_inputs(UpperCamelCase  )

											# XXX: adapt synced_gpus for fairscale as well
											# Priority (handled in generate):
											# gen_kwargs > model.generation_config > default GenerationConfig()

											if len(UpperCamelCase  ) == 0 and hasattr(self    ,		'_gen_kwargs'  ):
															snake_case__													=       self._gen_kwargs.copy()

											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.model.config.max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.model.config.num_beams
											)
											snake_case__													=       True if is_deepspeed_zeroa_enabled() else False
											snake_case__													=       (
											    gen_kwargs['synced_gpus'] if gen_kwargs.get('synced_gpus'  ) is not None else default_synced_gpus
											)

											# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
											# (otherwise, it would continue generating from the padded `decoder_input_ids`)
											if (
											    "labels" in inputs
											    and "decoder_input_ids" in inputs
											    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
											):
															snake_case__													=       {k: v for k, v in inputs.items() if k != 'decoder_input_ids'}
											snake_case__													=       self.model.generate(**UpperCamelCase    ,		**UpperCamelCase  )

											# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
											# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
											# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
											if self.model.generation_config._from_model_config:
															snake_case__													=       False

											# Retrieves GenerationConfig from model.generation_config
											snake_case__													=       self.model.generation_config
											# in case the batch is shorter than max length, the output should be padded
											if generated_tokens.shape[-1] < gen_config.max_length:
															snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_length  )
											elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
															snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_new_tokens + 1  )

											with torch.no_grad():
															if has_labels:
																			with self.compute_loss_context_manager():
																							snake_case__													=       model(**UpperCamelCase  )
																			if self.label_smoother is not None:
																							snake_case__													=       self.label_smoother(UpperCamelCase    ,		inputs['labels']  ).mean().detach()
																			else:
																							snake_case__													=       (outputs['loss'] if isinstance(UpperCamelCase    ,		UpperCamelCase  ) else outputs[0]).mean().detach()
															else:
																			snake_case__													=       None

											if self.args.prediction_loss_only:
															return loss, None, None

											if has_labels:
															snake_case__													=       inputs['labels']
															if labels.shape[-1] < gen_config.max_length:
																			snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_length  )
															elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																			snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_new_tokens + 1  )
											else:
															snake_case__													=       None

											return loss, generated_tokens, labels


							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Tuple  )						->						List[str]:
											if self.tokenizer is not None and hasattr(self.tokenizer    ,		'pad_token_id'  ):
															# If PAD token is not defined at least EOS token has to be defined
															snake_case__													=       (
															    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
															)
											else:
															if self.model.config.pad_token_id is not None:
																			snake_case__													=       self.model.config.pad_token_id
															else:
																			raise ValueError('Pad_token_id must be set in the configuration of the model, in order to pad tensors'  )

											snake_case__													=       pad_token_id * torch.ones(
											    (tensor.shape[0], max_length)    ,		dtype=tensor.dtype    ,		device=tensor.device  )
											snake_case__													=       tensor
											return padded_tensor




