






import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase   :						Optional[Any]							=  logging.get_logger(__name__)

__UpperCamelCase   :						Any							=  {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__UpperCamelCase   :						Any							=  {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__UpperCamelCase   :						str							=  {
    """Salesforce/codegen-350M-mono""": 2048,
}
@lru_cache()
def        a_	( )     ->     int:







				"""simple docstring"""
				snake_case__													=       (
				    list(range(ord('!'					) ,				ord('~'					) + 1					)					) + list(range(ord('¡'					) ,				ord('¬'					) + 1					)					) + list(range(ord('®'					) ,				ord('ÿ'					) + 1					)					)
				)
				snake_case__													=       bs[:]
				snake_case__													=       0
				for b in range(2**8					):
								if b not in bs:
												bs.append(_A					)
												cs.append(2**8 + n					)
												n += 1
				snake_case__													=       [chr(_A					) for n in cs]
				return dict(zip(_A ,				_A					)					)
def        a_	( _A					)     ->     int:







				"""simple docstring"""
				snake_case__													=       set()
				snake_case__													=       word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)					)
								snake_case__													=       char
				return pairs


class 					__SCREAMING_SNAKE_CASE(		lowerCAmelCase__		):
							_UpperCAmelCase	      =    VOCAB_FILES_NAMES
							_UpperCAmelCase	      =    PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase	      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase	      =    ['''input_ids''', '''attention_mask''']
							def __init__(       self: int    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: str="replace"    ,		UpperCamelCase: Dict="<|endoftext|>"    ,		UpperCamelCase: str="<|endoftext|>"    ,		UpperCamelCase: List[str]="<|endoftext|>"    ,		UpperCamelCase: Tuple=None    ,		UpperCamelCase: int=False    ,		UpperCamelCase: Union[str, Any]=False    ,		**UpperCamelCase: List[str]    ,		)						->						Optional[int]:
											snake_case__													=       AddedToken(__lowerCAmelCase    ,		lstrip=__lowerCAmelCase    ,		rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase    ,		__lowerCAmelCase  ) else bos_token
											snake_case__													=       AddedToken(__lowerCAmelCase    ,		lstrip=__lowerCAmelCase    ,		rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase    ,		__lowerCAmelCase  ) else eos_token
											snake_case__													=       AddedToken(__lowerCAmelCase    ,		lstrip=__lowerCAmelCase    ,		rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase    ,		__lowerCAmelCase  ) else unk_token
											snake_case__													=       AddedToken(__lowerCAmelCase    ,		lstrip=__lowerCAmelCase    ,		rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase    ,		__lowerCAmelCase  ) else pad_token
											super().__init__(
											    errors=__lowerCAmelCase    ,		unk_token=__lowerCAmelCase    ,		bos_token=__lowerCAmelCase    ,		eos_token=__lowerCAmelCase    ,		pad_token=__lowerCAmelCase    ,		add_prefix_space=__lowerCAmelCase    ,		add_bos_token=__lowerCAmelCase    ,		**__lowerCAmelCase    ,		)
											snake_case__													=       add_bos_token

											with open(__lowerCAmelCase    ,		encoding='utf-8'  ) as vocab_handle:
															snake_case__													=       json.load(__lowerCAmelCase  )
											snake_case__													=       {v: k for k, v in self.encoder.items()}
											snake_case__													=       errors  # how to handle errors in decoding
											snake_case__													=       bytes_to_unicode()
											snake_case__													=       {v: k for k, v in self.byte_encoder.items()}
											with open(__lowerCAmelCase    ,		encoding='utf-8'  ) as merges_handle:
															snake_case__													=       merges_handle.read().split('\n'  )[1:-1]
											snake_case__													=       [tuple(merge.split()  ) for merge in bpe_merges]
											snake_case__													=       dict(zip(__lowerCAmelCase    ,		range(len(__lowerCAmelCase  )  )  )  )
											snake_case__													=       {}
											snake_case__													=       add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											snake_case__													=       re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'  )
							@property
							def lowerCAmelCase_	(       self: List[str]  )						->						str:
											return len(self.encoder  )
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Optional[int]:
											return dict(self.encoder    ,		**self.added_tokens_encoder  )
							def lowerCAmelCase_	(       self: Dict    ,		UpperCamelCase: Optional[Any]  )						->						List[str]:


											if token in self.cache:
															return self.cache[token]
											snake_case__													=       tuple(__lowerCAmelCase  )
											snake_case__													=       get_pairs(__lowerCAmelCase  )

											if not pairs:
															return token

											while True:
															snake_case__													=       min(__lowerCAmelCase    ,		key=lambda UpperCamelCase  : self.bpe_ranks.get(__lowerCAmelCase    ,		float('inf'  )  )  )
															if bigram not in self.bpe_ranks:
																			break
															snake_case__							,  snake_case__													=       bigram
															snake_case__													=       []
															snake_case__													=       0
															while i < len(__lowerCAmelCase  ):
																			try:
																							snake_case__													=       word.index(__lowerCAmelCase    ,		__lowerCAmelCase  )
																			except ValueError:
																							new_word.extend(word[i:]  )
																							break
																			else:
																							new_word.extend(word[i:j]  )
																							snake_case__													=       j

																			if word[i] == first and i < len(__lowerCAmelCase  ) - 1 and word[i + 1] == second:
																							new_word.append(first + second  )
																							i += 2
																			else:
																							new_word.append(word[i]  )
																							i += 1
															snake_case__													=       tuple(__lowerCAmelCase  )
															snake_case__													=       new_word
															if len(__lowerCAmelCase  ) == 1:
																			break
															else:
																			snake_case__													=       get_pairs(__lowerCAmelCase  )
											snake_case__													=       ' '.join(__lowerCAmelCase  )
											snake_case__													=       word
											return word
							def lowerCAmelCase_	(       self: Dict    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: Dict=None  )						->						Tuple:


											if self.add_bos_token:
															snake_case__													=       [self.bos_token_id]
											else:
															snake_case__													=       []

											snake_case__													=       bos_token_ids + token_ids_a

											if token_ids_a is None:
															return output

											return output + bos_token_ids + token_ids_a
							def lowerCAmelCase_	(       self: int    ,		UpperCamelCase: Dict  )						->						Any:
											snake_case__													=       []
											for token in re.findall(self.pat    ,		__lowerCAmelCase  ):
															snake_case__													=       ''.join(
															    self.byte_encoder[b] for b in token.encode('utf-8'  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase  ).split(' '  )  )
											return bpe_tokens
							def lowerCAmelCase_	(       self: Optional[int]    ,		UpperCamelCase: int  )						->						Tuple:
											return self.encoder.get(__lowerCAmelCase    ,		self.encoder.get(self.unk_token  )  )
							def lowerCAmelCase_	(       self: Optional[int]    ,		UpperCamelCase: Dict  )						->						List[str]:
											return self.decoder.get(__lowerCAmelCase  )
							def lowerCAmelCase_	(       self: List[str]    ,		UpperCamelCase: Optional[int]  )						->						Optional[Any]:
											snake_case__													=       ''.join(__lowerCAmelCase  )
											snake_case__													=       bytearray([self.byte_decoder[c] for c in text]  ).decode('utf-8'    ,		errors=self.errors  )
											return text
							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: str    ,		UpperCamelCase: Tuple = None  )						->						List[Any]:


											if not os.path.isdir(__lowerCAmelCase  ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
															return
											snake_case__													=       os.path.join(
											    __lowerCAmelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
											snake_case__													=       os.path.join(
											    __lowerCAmelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

											with open(__lowerCAmelCase    ,		'w'    ,		encoding='utf-8'  ) as f:
															f.write(json.dumps(self.encoder    ,		indent=2    ,		sort_keys=__lowerCAmelCase    ,		ensure_ascii=__lowerCAmelCase  ) + '\n'  )

											snake_case__													=       0
											with open(__lowerCAmelCase    ,		'w'    ,		encoding='utf-8'  ) as writer:
															writer.write('#version: 0.2\n'  )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,		key=lambda UpperCamelCase  : kv[1]  ):
																			if index != token_index:
																							logger.warning(
																							    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																							    ' Please check that the tokenizer is not corrupted!'  )
																							snake_case__													=       token_index
																			writer.write(' '.join(__lowerCAmelCase  ) + '\n'  )
																			index += 1

											return vocab_file, merge_file
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: Optional[Any]=False    ,		**UpperCamelCase: Any  )						->						Tuple:
											snake_case__													=       kwargs.pop('add_prefix_space'    ,		self.add_prefix_space  )
											if is_split_into_words or add_prefix_space:
															snake_case__													=       ' ' + text
											return (text, kwargs)
							def lowerCAmelCase_	(       self: Any    ,		UpperCamelCase: Any    ,		UpperCamelCase: str = False    ,		UpperCamelCase: Any = None    ,		UpperCamelCase: int = None    ,		**UpperCamelCase: Any    ,		)						->						Union[str, Any]:
											snake_case__													=       super()._decode(
											    token_ids=__lowerCAmelCase    ,		skip_special_tokens=__lowerCAmelCase    ,		clean_up_tokenization_spaces=__lowerCAmelCase    ,		**__lowerCAmelCase    ,		)

											if truncate_before_pattern is not None and len(__lowerCAmelCase  ) > 0:
															snake_case__													=       self.truncate(__lowerCAmelCase    ,		__lowerCAmelCase  )

											return decoded_text


							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: Dict  )						->						Any:


											def find_re(UpperCamelCase: Optional[int]    ,		UpperCamelCase: Any    ,		UpperCamelCase: List[Any]  ):
															snake_case__													=       pattern.search(__lowerCAmelCase    ,		__lowerCAmelCase  )
															return m.start() if m else -1

											snake_case__													=       [re.compile(__lowerCAmelCase    ,		re.MULTILINE  ) for pattern in truncate_before_pattern]

											snake_case__													=       list(re.finditer('^print'    ,		__lowerCAmelCase    ,		re.MULTILINE  )  )

											if len(__lowerCAmelCase  ) > 1:
															snake_case__													=       completion[: prints[1].start()]

											snake_case__													=       list(re.finditer('^def'    ,		__lowerCAmelCase    ,		re.MULTILINE  )  )

											if len(__lowerCAmelCase  ) > 1:
															snake_case__													=       completion[: defs[1].start()]

											snake_case__													=       0

											snake_case__													=       [
											    pos for pos in [find_re(__lowerCAmelCase    ,		__lowerCAmelCase    ,		__lowerCAmelCase  ) for terminal in terminals] if pos != -1
											]

											if len(__lowerCAmelCase  ) > 0:
															return completion[: min(__lowerCAmelCase  )]
											else:
															return completion












from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
							from .data.data_collator import DataCollator
							from .modeling_utils import PreTrainedModel
							from .tokenization_utils_base import PreTrainedTokenizerBase
							from .trainer_callback import TrainerCallback
							from .trainer_utils import EvalPrediction, PredictionOutput
							from .training_args import TrainingArguments


__UpperCamelCase   :						Tuple							=  logging.get_logger(__name__)


class 					__SCREAMING_SNAKE_CASE(		a_		):
							def __init__(       self: Dict    ,		UpperCamelCase: Union["PreTrainedModel", nn.Module] = None    ,		UpperCamelCase: "TrainingArguments" = None    ,		UpperCamelCase: Optional["DataCollator"] = None    ,		UpperCamelCase: Optional[Dataset] = None    ,		UpperCamelCase: Optional[Union[Dataset, Dict[str, Dataset]]] = None    ,		UpperCamelCase: Optional["PreTrainedTokenizerBase"] = None    ,		UpperCamelCase: Optional[Callable[[], "PreTrainedModel"]] = None    ,		UpperCamelCase: Optional[Callable[["EvalPrediction"], Dict]] = None    ,		UpperCamelCase: Optional[List["TrainerCallback"]] = None    ,		UpperCamelCase: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None)    ,		UpperCamelCase: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None    ,		)						->						Tuple:
											super().__init__(
											    model=UpperCamelCase    ,		args=UpperCamelCase    ,		data_collator=UpperCamelCase    ,		train_dataset=UpperCamelCase    ,		eval_dataset=UpperCamelCase    ,		tokenizer=UpperCamelCase    ,		model_init=UpperCamelCase    ,		compute_metrics=UpperCamelCase    ,		callbacks=UpperCamelCase    ,		optimizers=UpperCamelCase    ,		preprocess_logits_for_metrics=UpperCamelCase    ,		)

											# Override self.model.generation_config if a GenerationConfig is specified in args.
											# Priority: args.generation_config > model.generation_config > default GenerationConfig.
											if self.args.generation_config is not None:
															snake_case__													=       self.load_generation_config(self.args.generation_config  )
															snake_case__													=       gen_config
							@staticmethod
							def lowerCAmelCase_	(       UpperCamelCase: Union[str, GenerationConfig]  )						->						GenerationConfig:

											# GenerationConfig provided, nothing to do
											if isinstance(UpperCamelCase    ,		UpperCamelCase  ):
															return deepcopy(UpperCamelCase  )

											# str or Path
											snake_case__													=       Path(UpperCamelCase  ) if isinstance(UpperCamelCase    ,		UpperCamelCase  ) else gen_config_arg
											snake_case__													=       None

											# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
											# This step is required in order to determine config_file_name
											if pretrained_model_name.is_file():
															snake_case__													=       pretrained_model_name.name
															snake_case__													=       pretrained_model_name.parent
											# dir path
											elif pretrained_model_name.is_dir():
															pass
											# model id or URL
											else:
															snake_case__													=       gen_config_arg

											snake_case__													=       GenerationConfig.from_pretrained(UpperCamelCase    ,		UpperCamelCase  )
											return gen_config
							def lowerCAmelCase_	(       self: Optional[int]    ,		UpperCamelCase: Optional[Dataset] = None    ,		UpperCamelCase: Optional[List[str]] = None    ,		UpperCamelCase: str = "eval"    ,		**UpperCamelCase: Dict    ,		)						->						Dict[str, float]:
											snake_case__													=       gen_kwargs.copy()
											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.args.generation_max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.args.generation_num_beams
											)
											snake_case__													=       gen_kwargs

											return super().evaluate(UpperCamelCase    ,		ignore_keys=UpperCamelCase    ,		metric_key_prefix=UpperCamelCase  )
							def lowerCAmelCase_	(       self: Tuple    ,		UpperCamelCase: Dataset    ,		UpperCamelCase: Optional[List[str]] = None    ,		UpperCamelCase: str = "test"    ,		**UpperCamelCase: Dict    ,		)						->						"PredictionOutput":
											snake_case__													=       gen_kwargs.copy()
											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.args.generation_max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.args.generation_num_beams
											)
											snake_case__													=       gen_kwargs

											return super().predict(UpperCamelCase    ,		ignore_keys=UpperCamelCase    ,		metric_key_prefix=UpperCamelCase  )
							def lowerCAmelCase_	(       self: Tuple    ,		UpperCamelCase: nn.Module    ,		UpperCamelCase: Dict[str, Union[torch.Tensor, Any]]    ,		UpperCamelCase: bool    ,		UpperCamelCase: Optional[List[str]] = None    ,		**UpperCamelCase: List[Any]    ,		)						->						Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:

											if not self.args.predict_with_generate or prediction_loss_only:
															return super().prediction_step(
															    UpperCamelCase    ,		UpperCamelCase    ,		prediction_loss_only=UpperCamelCase    ,		ignore_keys=UpperCamelCase  )

											snake_case__													=       'labels' in inputs
											snake_case__													=       self._prepare_inputs(UpperCamelCase  )

											# XXX: adapt synced_gpus for fairscale as well
											# Priority (handled in generate):
											# gen_kwargs > model.generation_config > default GenerationConfig()

											if len(UpperCamelCase  ) == 0 and hasattr(self    ,		'_gen_kwargs'  ):
															snake_case__													=       self._gen_kwargs.copy()

											if gen_kwargs.get('max_length'  ) is None and gen_kwargs.get('max_new_tokens'  ) is None:
															snake_case__													=       self.model.config.max_length
											snake_case__													=       (
											    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'  ) is not None else self.model.config.num_beams
											)
											snake_case__													=       True if is_deepspeed_zeroa_enabled() else False
											snake_case__													=       (
											    gen_kwargs['synced_gpus'] if gen_kwargs.get('synced_gpus'  ) is not None else default_synced_gpus
											)

											# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
											# (otherwise, it would continue generating from the padded `decoder_input_ids`)
											if (
											    "labels" in inputs
											    and "decoder_input_ids" in inputs
											    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
											):
															snake_case__													=       {k: v for k, v in inputs.items() if k != 'decoder_input_ids'}
											snake_case__													=       self.model.generate(**UpperCamelCase    ,		**UpperCamelCase  )

											# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
											# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
											# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
											if self.model.generation_config._from_model_config:
															snake_case__													=       False

											# Retrieves GenerationConfig from model.generation_config
											snake_case__													=       self.model.generation_config
											# in case the batch is shorter than max length, the output should be padded
											if generated_tokens.shape[-1] < gen_config.max_length:
															snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_length  )
											elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
															snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_new_tokens + 1  )

											with torch.no_grad():
															if has_labels:
																			with self.compute_loss_context_manager():
																							snake_case__													=       model(**UpperCamelCase  )
																			if self.label_smoother is not None:
																							snake_case__													=       self.label_smoother(UpperCamelCase    ,		inputs['labels']  ).mean().detach()
																			else:
																							snake_case__													=       (outputs['loss'] if isinstance(UpperCamelCase    ,		UpperCamelCase  ) else outputs[0]).mean().detach()
															else:
																			snake_case__													=       None

											if self.args.prediction_loss_only:
															return loss, None, None

											if has_labels:
															snake_case__													=       inputs['labels']
															if labels.shape[-1] < gen_config.max_length:
																			snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_length  )
															elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																			snake_case__													=       self._pad_tensors_to_max_len(UpperCamelCase    ,		gen_config.max_new_tokens + 1  )
											else:
															snake_case__													=       None

											return loss, generated_tokens, labels


							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Tuple  )						->						List[str]:
											if self.tokenizer is not None and hasattr(self.tokenizer    ,		'pad_token_id'  ):
															# If PAD token is not defined at least EOS token has to be defined
															snake_case__													=       (
															    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
															)
											else:
															if self.model.config.pad_token_id is not None:
																			snake_case__													=       self.model.config.pad_token_id
															else:
																			raise ValueError('Pad_token_id must be set in the configuration of the model, in order to pad tensors'  )

											snake_case__													=       pad_token_id * torch.ones(
											    (tensor.shape[0], max_length)    ,		dtype=tensor.dtype    ,		device=tensor.device  )
											snake_case__													=       tensor
											return padded_tensor




