


from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
						import numpy as np
						import tensorflow as tf

						from transformers import (
						    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
						    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_MASKED_LM_MAPPING,
						    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
						    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
						    TF_MODEL_FOR_PRETRAINING_MAPPING,
						    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
						    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
						    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
						    TFSharedEmbeddings,
						)

						if _tf_gpu_memory_limit is not None:
												__UpperCamelCase      						=    tf.config.list_physical_devices('GPU')
												for gpu in gpus:
																		# Restrict TensorFlow to only allocate x GB of memory on the GPUs
																		try:
																								tf.config.set_logical_device_configuration(
																								    gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
																								)
																								__UpperCamelCase      						=    tf.config.list_logical_devices('GPU')
																								print('Logical GPUs', logical_gpus)
																		except RuntimeError as e:
																								# Virtual devices must be set before GPUs have been initialized
																								print(e)







@require_tf
class     lowerCamelCase__		:






			"""simple docstring"""






			_UpperCamelCase						:   Dict					=						None
			_UpperCamelCase						:   Optional[int]					=						()
			_UpperCamelCase						:   Any					=						()
			_UpperCamelCase						:   Optional[Any]					=						True
			_UpperCamelCase						:   str					=						True
			_UpperCamelCase						:   Any					=						True
			_UpperCamelCase						:   Optional[int]					=						False




			def 			snake_case__			(							self ,  snake_case ,  snake_case ,  snake_case=False       ):

						'''simple docstring'''


						UpperCamelCase__   =    copy.deepcopy(snake_case       )

						if model_class in get_values(snake_case       ):
									UpperCamelCase__   =    {
									    k: tf.tile(tf.expand_dims(snake_case ,  1       ) ,  (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)       )
									    if isinstance(snake_case ,  tf.Tensor       ) and v.ndim > 0
									    else v
									    for k, v in inputs_dict.items()
									}

						if return_labels:
									if model_class in get_values(snake_case       ):
												UpperCamelCase__   =    tf.ones(self.model_tester.batch_size ,  dtype=tf.intaa       )
									elif model_class in get_values(snake_case       ):
												UpperCamelCase__   =    tf.zeros(self.model_tester.batch_size ,  dtype=tf.intaa       )
												UpperCamelCase__   =    tf.zeros(self.model_tester.batch_size ,  dtype=tf.intaa       )
									elif model_class in [
									    *get_values(snake_case       ),
									    *get_values(snake_case       ),
									]:
												UpperCamelCase__   =    tf.zeros(self.model_tester.batch_size ,  dtype=tf.intaa       )
									elif model_class in get_values(snake_case       ):
												UpperCamelCase__   =    tf.zeros(self.model_tester.batch_size ,  dtype=tf.intaa       )
									elif model_class in [
									    *get_values(snake_case       ),
									    *get_values(snake_case       ),
									    *get_values(snake_case       ),
									    *get_values(snake_case       ),
									    *get_values(snake_case       ),
									]:
												UpperCamelCase__   =    tf.zeros(
												    (self.model_tester.batch_size, self.model_tester.seq_length) ,  dtype=tf.intaa       )
						return inputs_dict




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes[:2]:
									UpperCamelCase__   =    self._prepare_for_class(snake_case ,  snake_case       )
									UpperCamelCase__   =    model_class(snake_case       )

									@tf.function
									def run_in_graph_mode():
												return model(snake_case       )

									UpperCamelCase__   =    run_in_graph_mode()
									self.assertIsNotNone(snake_case       )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes[:2]:
									UpperCamelCase__   =    self._prepare_for_class(snake_case ,  snake_case       )
									UpperCamelCase__   =    model_class(snake_case       )

									@tf.function(experimental_compile=snake_case       )
									def run_in_graph_mode():
												return model(snake_case       )

									UpperCamelCase__   =    run_in_graph_mode()
									self.assertIsNotNone(snake_case       )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes[:2]:
									UpperCamelCase__   =    model_class(snake_case       )
									if getattr(snake_case ,  "hf_compute_loss" ,  snake_case       ):
												# Test that model correctly compute the loss with kwargs
												UpperCamelCase__   =    self._prepare_for_class(inputs_dict.copy() ,  snake_case ,  return_labels=snake_case       )
												# Is there a better way to remove these decoder inputs?
												UpperCamelCase__   =    {
												    key: val
												    for key, val in prepared_for_class.items()
												    if key not in ("head_mask", "decoder_head_mask", "cross_attn_head_mask", "decoder_input_ids")
												}

												UpperCamelCase__   =    {
												    "labels",
												    "label",
												    "label_ids",
												    "start_positions",
												    "start_position",
												    "end_positions",
												    "end_position",
												    "next_sentence_label",
												}
												UpperCamelCase__   =    possible_label_cols.intersection(set(snake_case       )       )
												self.assertGreater(len(snake_case       ) ,  0 ,  msg="No matching label names found!"       )
												UpperCamelCase__   =    {key: val for key, val in prepared_for_class.items() if key in label_names}
												UpperCamelCase__   =    {key: val for key, val in prepared_for_class.items() if key not in label_names}
												self.assertGreater(len(snake_case       ) ,  0       )

												# Make sure it works with XLA!
												model.compile(optimizer=tf.keras.optimizers.SGD(0.0       ) ,  jit_compile=snake_case       )
												# Make sure the model fits without crashing regardless of where we pass the labels
												UpperCamelCase__   =    model.fit(
												    snake_case ,  validation_data=snake_case ,  steps_per_epoch=1 ,  validation_steps=1 ,  shuffle=snake_case ,  verbose=0 ,  )
												UpperCamelCase__   =    history.history["loss"][0]
												self.assertTrue(not isnan(snake_case       )       )
												UpperCamelCase__   =    history.history["val_loss"][0]
												self.assertTrue(not isnan(snake_case       )       )

												# Now test it with separate labels, to make sure that path works in XLA too.
												UpperCamelCase__   =    model_class(snake_case       )
												model.compile(optimizer=tf.keras.optimizers.SGD(0.0       ) ,  jit_compile=snake_case       )
												UpperCamelCase__   =    model.fit(
												    snake_case ,  snake_case ,  validation_data=(inputs_minus_labels, labels) ,  steps_per_epoch=1 ,  validation_steps=1 ,  shuffle=snake_case ,  verbose=0 ,  )

												UpperCamelCase__   =    history.history["loss"][0]
												self.assertTrue(not isnan(snake_case       )       )
												UpperCamelCase__   =    history.history["val_loss"][0]
												self.assertTrue(not isnan(snake_case       )       )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
						UpperCamelCase__   =    True
						UpperCamelCase__   =    True

						if hasattr(snake_case ,  "use_cache"       ):
									UpperCamelCase__   =    True

						UpperCamelCase__   =    getattr(self.model_tester ,  "encoder_seq_length" ,  self.model_tester.seq_length       )
						UpperCamelCase__   =    getattr(self.model_tester ,  "key_length" ,  snake_case       )

						for model_class in self.all_model_classes[:2]:
									UpperCamelCase__   =    self._prepare_for_class(snake_case ,  snake_case       )
									UpperCamelCase__   =    model_class(snake_case       )
									model.build()
									UpperCamelCase__   =    len(model(snake_case       )       )

									for key in list(class_inputs_dict.keys()       ):
												# Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
												if key not in model.input_signature:
															del class_inputs_dict[key]
												# Check it's a tensor, in case the inputs dict has some bools in it too
												elif isinstance(class_inputs_dict[key] ,  tf.Tensor       ) and class_inputs_dict[key].dtype.is_integer:
															UpperCamelCase__   =    tf.cast(class_inputs_dict[key] ,  tf.intaa       )

									if set(class_inputs_dict.keys()       ) != set(model.input_signature.keys()       ):
												continue  # Some models have inputs that the preparation functions don't create, we skip those

									with tempfile.TemporaryDirectory() as tmpdirname:
												model.save_pretrained(snake_case ,  saved_model=snake_case       )
												UpperCamelCase__   =    os.path.join(snake_case ,  "saved_model" ,  "1"       )
												UpperCamelCase__   =    tf.keras.models.load_model(snake_case       )
												UpperCamelCase__   =    model(snake_case       )

												if self.is_encoder_decoder:
															UpperCamelCase__   =    outputs["encoder_hidden_states"]
															UpperCamelCase__   =    outputs["encoder_attentions"]
												else:
															UpperCamelCase__   =    outputs["hidden_states"]
															UpperCamelCase__   =    outputs["attentions"]

												self.assertEqual(len(snake_case       ) ,  snake_case       )

												UpperCamelCase__   =    getattr(
												    self.model_tester ,  "expected_num_hidden_layers" ,  self.model_tester.num_hidden_layers + 1       )

												self.assertEqual(len(snake_case       ) ,  snake_case       )
												self.assertListEqual(
												    list(output_hidden_states[0].shape[-2:]       ) ,  [self.model_tester.seq_length, self.model_tester.hidden_size] ,  )

												self.assertEqual(len(snake_case       ) ,  self.model_tester.num_hidden_layers       )
												self.assertListEqual(
												    list(output_attentions[0].shape[-3:]       ) ,  [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length] ,  )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						tf.keras.mixed_precision.set_global_policy("mixed_float16"       )

						# try/finally block to ensure subsequent tests run in float32
						try:
									UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
									for model_class in self.all_model_classes[:2]:
												UpperCamelCase__   =    self._prepare_for_class(snake_case ,  snake_case       )
												UpperCamelCase__   =    model_class(snake_case       )
												UpperCamelCase__   =    model(snake_case       )

												self.assertIsNotNone(snake_case       )
						finally:
									tf.keras.mixed_precision.set_global_policy("float32"       )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()
						# head_mask and decoder_head_mask has different shapes than other input args
						if "head_mask" in inputs_dict:
									del inputs_dict["head_mask"]
						if "decoder_head_mask" in inputs_dict:
									del inputs_dict["decoder_head_mask"]
						if "cross_attn_head_mask" in inputs_dict:
									del inputs_dict["cross_attn_head_mask"]
						UpperCamelCase__   =    {
						    module_member
						    for model_class in self.all_model_classes
						    for module in (import_module(model_class.__module__       ),)
						    for module_member_name in dir(snake_case       )
						    if module_member_name.endswith("MainLayer"       )
						    for module_member in (getattr(snake_case ,  snake_case       ),)
						    if isinstance(snake_case ,  snake_case       )
						    and tf.keras.layers.Layer in module_member.__bases__
						    and getattr(snake_case ,  "_keras_serializable" ,  snake_case       )
						}

						for main_layer_class in tf_main_layer_classes:
									# T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
									if "T5" in main_layer_class.__name__:
												# Take the same values than in TFT5ModelTester for this shared layer
												UpperCamelCase__   =    TFSharedEmbeddings(self.model_tester.vocab_size ,  self.model_tester.hidden_size ,  name="shared"       )
												UpperCamelCase__   =    False
												UpperCamelCase__   =    main_layer_class(snake_case ,  embed_tokens=snake_case       )
									else:
												UpperCamelCase__   =    main_layer_class(snake_case       )

									UpperCamelCase__   =    {
									    name: tf.keras.Input(tensor.shape[1:] ,  dtype=tensor.dtype       ) for name, tensor in inputs_dict.items()
									}

									if hasattr(self.model_tester ,  "num_labels"       ):
												UpperCamelCase__   =    self.model_tester.num_labels
									else:
												UpperCamelCase__   =    2

									UpperCamelCase__   =    tf.data.Dataset.from_tensor_slices(
									    (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)       ))       ).batch(1       )

									UpperCamelCase__   =    main_layer(snake_case       )[0]
									UpperCamelCase__   =    tf.keras.layers.Dense(snake_case ,  activation="softmax" ,  name="outputs"       )(snake_case       )
									UpperCamelCase__   =    tf.keras.models.Model(inputs=snake_case ,  outputs=[outputs]       )

									model.compile(loss="binary_crossentropy" ,  optimizer="adam" ,  metrics=["binary_accuracy"]       )
									model.fit(snake_case ,  epochs=1       )

									with tempfile.TemporaryDirectory() as tmpdirname:
												UpperCamelCase__   =    os.path.join(snake_case ,  "keras_model.h5"       )
												model.save(snake_case       )
												if "T5" in main_layer_class.__name__:
															UpperCamelCase__   =    tf.keras.models.load_model(
															    snake_case ,  custom_objects={
															        main_layer_class.__name__: main_layer_class,
															        "TFSharedEmbeddings": TFSharedEmbeddings,
															    } ,  )
												else:
															UpperCamelCase__   =    tf.keras.models.load_model(
															    snake_case ,  custom_objects={main_layer_class.__name__: main_layer_class}       )
												assert isinstance(snake_case ,  tf.keras.Model       )
												model(snake_case       )




			@slow
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						UpperCamelCase__, UpperCamelCase__   =    self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes[:2]:
									UpperCamelCase__   =    model_class(snake_case       )

									UpperCamelCase__   =    copy.deepcopy(snake_case       )

									if not self.is_encoder_decoder:
												UpperCamelCase__   =    inputs["input_ids"]
												del inputs["input_ids"]
									else:
												UpperCamelCase__   =    inputs["input_ids"]
												UpperCamelCase__   =    inputs.get("decoder_input_ids" ,  snake_case       )
												del inputs["input_ids"]
												inputs.pop("decoder_input_ids" ,  snake_case       )

									if not self.is_encoder_decoder:
												UpperCamelCase__   =    model.get_input_embeddings()(snake_case       )
									else:
												UpperCamelCase__   =    model.get_input_embeddings()(snake_case       )
												UpperCamelCase__   =    model.get_input_embeddings()(snake_case       )

									UpperCamelCase__   =    self._prepare_for_class(snake_case ,  snake_case       )

									@tf.function
									def run_in_graph_mode():
												return model(snake_case       )

									UpperCamelCase__   =    run_in_graph_mode()
									self.assertIsNotNone(snake_case       )




			def 			snake_case__			(							self ,  snake_case ,  snake_case       ):

						'''simple docstring'''


						UpperCamelCase__   =    []
						if model.config.bos_token_id is not None:
									special_tokens.append(model.config.bos_token_id       )
						if model.config.pad_token_id is not None:
									special_tokens.append(model.config.pad_token_id       )
						if model.config.eos_token_id is not None:
									special_tokens.append(model.config.eos_token_id       )

						# create random bad tokens that are not special tokens
						UpperCamelCase__   =    []
						while len(snake_case       ) < num_bad_tokens:
									UpperCamelCase__   =    tf.squeeze(ids_tensor((1, 1) ,  self.model_tester.vocab_size       ) ,  0       ).numpy()[0]
									if token not in special_tokens:
												bad_tokens.append(snake_case       )
						return bad_tokens




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						for token_id in output_ids[0].numpy().tolist():
									self.assertGreaterEqual(snake_case ,  0       )
									self.assertLess(snake_case ,  self.model_tester.vocab_size       )







			def 			snake_case__			(							self ,  snake_case ,  snake_case       ):

						'''simple docstring'''


						for bad_word_ids in bad_words_ids:
									# for all slices in batch
									for generated_ids_slice in generated_ids:
												# for all word idx
												for i in range(len(snake_case       ) ,  len(snake_case       )       ):
															# if tokens match
															if generated_ids_slice[i - len(snake_case       ) : i] == bad_word_ids:
																		return True
						return False






import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase      						=    logging.get_logger(__name__)

__UpperCamelCase      						=    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__UpperCamelCase      						=    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

__UpperCamelCase      						=    {
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}
@lru_cache()
def     UpperCamelCase_(			)->						Tuple:
			UpperCamelCase__   =    (
			    list(range(ord("!"					)			,       ord("~"					) + 1					)					) + list(range(ord("¡"					)			,       ord("¬"					) + 1					)					) + list(range(ord("®"					)			,       ord("ÿ"					) + 1					)					)
			)
			UpperCamelCase__   =    bs[:]
			UpperCamelCase__   =    0
			for b in range(2**8					):
						if b not in bs:
									bs.append(_A					)
									cs.append(2**8 + n					)
									n += 1
			UpperCamelCase__   =    [chr(_A					) for n in cs]
			return dict(zip(_A			,       _A					)					)
def     UpperCamelCase_(			_A				:Tuple					)->						Optional[Any]:
			UpperCamelCase__   =    set()
			UpperCamelCase__   =    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)					)
						UpperCamelCase__   =    char
			return pairs







class     lowerCamelCase__		(							UpperCAmelCase ):






			"""simple docstring"""






			_UpperCamelCase						:   Optional[Any]					=						VOCAB_FILES_NAMES
			_UpperCamelCase						:   List[Any]					=						PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase						:   Any					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase						:   int					=						['input_ids', 'attention_mask']




			def __init__(							self ,  snake_case ,  snake_case ,  snake_case="replace" ,  snake_case="<|endoftext|>" ,  snake_case="<|endoftext|>" ,  snake_case="<|endoftext|>" ,  snake_case=None ,  snake_case=False ,  snake_case=False ,  **snake_case ,  ):

						'''simple docstring'''


						UpperCamelCase__   =    AddedToken(snake_case ,  lstrip=snake_case ,  rstrip=snake_case       ) if isinstance(snake_case ,  snake_case       ) else bos_token
						UpperCamelCase__   =    AddedToken(snake_case ,  lstrip=snake_case ,  rstrip=snake_case       ) if isinstance(snake_case ,  snake_case       ) else eos_token
						UpperCamelCase__   =    AddedToken(snake_case ,  lstrip=snake_case ,  rstrip=snake_case       ) if isinstance(snake_case ,  snake_case       ) else unk_token
						UpperCamelCase__   =    AddedToken(snake_case ,  lstrip=snake_case ,  rstrip=snake_case       ) if isinstance(snake_case ,  snake_case       ) else pad_token
						super().__init__(
						    errors=snake_case ,  unk_token=snake_case ,  bos_token=snake_case ,  eos_token=snake_case ,  pad_token=snake_case ,  add_prefix_space=snake_case ,  add_bos_token=snake_case ,  **snake_case ,  )
						UpperCamelCase__   =    add_bos_token

						with open(snake_case ,  encoding="utf-8"       ) as vocab_handle:
									UpperCamelCase__   =    json.load(snake_case       )
						UpperCamelCase__   =    {v: k for k, v in self.encoder.items()}
						UpperCamelCase__   =    errors  # how to handle errors in decoding
						UpperCamelCase__   =    bytes_to_unicode()
						UpperCamelCase__   =    {v: k for k, v in self.byte_encoder.items()}
						with open(snake_case ,  encoding="utf-8"       ) as merges_handle:
									UpperCamelCase__   =    merges_handle.read().split("\n"       )[1:-1]
						UpperCamelCase__   =    [tuple(merge.split()       ) for merge in bpe_merges]
						UpperCamelCase__   =    dict(zip(snake_case ,  range(len(snake_case       )       )       )       )
						UpperCamelCase__   =    {}
						UpperCamelCase__   =    add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						UpperCamelCase__   =    re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"       )




			@property
			def 			snake_case__			(							self       ):

						'''simple docstring'''


						return len(self.encoder       )




			def 			snake_case__			(							self       ):

						'''simple docstring'''


						return dict(self.encoder ,  **self.added_tokens_encoder       )




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						if token in self.cache:
									return self.cache[token]
						UpperCamelCase__   =    tuple(snake_case       )
						UpperCamelCase__   =    get_pairs(snake_case       )

						if not pairs:
									return token

						while True:
									UpperCamelCase__   =    min(snake_case ,  key=lambda snake_case       : self.bpe_ranks.get(snake_case ,  float("inf"       )       )       )
									if bigram not in self.bpe_ranks:
												break
									UpperCamelCase__, UpperCamelCase__   =    bigram
									UpperCamelCase__   =    []
									UpperCamelCase__   =    0
									while i < len(snake_case       ):
												try:
															UpperCamelCase__   =    word.index(snake_case ,  snake_case       )
												except ValueError:
															new_word.extend(word[i:]       )
															break
												else:
															new_word.extend(word[i:j]       )
															UpperCamelCase__   =    j

												if word[i] == first and i < len(snake_case       ) - 1 and word[i + 1] == second:
															new_word.append(first + second       )
															i += 2
												else:
															new_word.append(word[i]       )
															i += 1
									UpperCamelCase__   =    tuple(snake_case       )
									UpperCamelCase__   =    new_word
									if len(snake_case       ) == 1:
												break
									else:
												UpperCamelCase__   =    get_pairs(snake_case       )
						UpperCamelCase__   =    " ".join(snake_case       )
						UpperCamelCase__   =    word
						return word




			def 			snake_case__			(							self ,  snake_case ,  snake_case=None       ):

						'''simple docstring'''


						if self.add_bos_token:
									UpperCamelCase__   =    [self.bos_token_id]
						else:
									UpperCamelCase__   =    []

						UpperCamelCase__   =    bos_token_ids + token_ids_a

						if token_ids_a is None:
									return output

						return output + bos_token_ids + token_ids_a




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						UpperCamelCase__   =    []
						for token in re.findall(self.pat ,  snake_case       ):
									UpperCamelCase__   =    "".join(
									    self.byte_encoder[b] for b in token.encode("utf-8"       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case       ).split(" "       )       )
						return bpe_tokens




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						return self.encoder.get(snake_case ,  self.encoder.get(self.unk_token       )       )




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						return self.decoder.get(snake_case       )




			def 			snake_case__			(							self ,  snake_case       ):

						'''simple docstring'''


						UpperCamelCase__   =    "".join(snake_case       )
						UpperCamelCase__   =    bytearray([self.byte_decoder[c] for c in text]       ).decode("utf-8" ,  errors=self.errors       )
						return text




			def 			snake_case__			(							self ,  snake_case ,  snake_case = None       ):

						'''simple docstring'''


						if not os.path.isdir(snake_case       ):
									logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
									return
						UpperCamelCase__   =    os.path.join(
						    snake_case ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
						UpperCamelCase__   =    os.path.join(
						    snake_case ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

						with open(snake_case ,  "w" ,  encoding="utf-8"       ) as f:
									f.write(json.dumps(self.encoder ,  indent=2 ,  sort_keys=snake_case ,  ensure_ascii=snake_case       ) + "\n"       )

						UpperCamelCase__   =    0
						with open(snake_case ,  "w" ,  encoding="utf-8"       ) as writer:
									writer.write("#version: 0.2\n"       )
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,  key=lambda snake_case       : kv[1]       ):
												if index != token_index:
															logger.warning(
															    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
															    " Please check that the tokenizer is not corrupted!"       )
															UpperCamelCase__   =    token_index
												writer.write(" ".join(snake_case       ) + "\n"       )
												index += 1

						return vocab_file, merge_file




			def 			snake_case__			(							self ,  snake_case ,  snake_case=False ,  **snake_case       ):

						'''simple docstring'''


						UpperCamelCase__   =    kwargs.pop("add_prefix_space" ,  self.add_prefix_space       )
						if is_split_into_words or add_prefix_space:
									UpperCamelCase__   =    " " + text
						return (text, kwargs)




			def 			snake_case__			(							self ,  snake_case ,  snake_case = False ,  snake_case = None ,  snake_case = None ,  **snake_case ,  ):

						'''simple docstring'''


						UpperCamelCase__   =    super()._decode(
						    token_ids=snake_case ,  skip_special_tokens=snake_case ,  clean_up_tokenization_spaces=snake_case ,  **snake_case ,  )

						if truncate_before_pattern is not None and len(snake_case       ) > 0:
									UpperCamelCase__   =    self.truncate(snake_case ,  snake_case       )

						return decoded_text







			def 			snake_case__			(							self ,  snake_case ,  snake_case       ):

						'''simple docstring'''


						def find_re(snake_case ,  snake_case ,  snake_case       ):
									UpperCamelCase__   =    pattern.search(snake_case ,  snake_case       )
									return m.start() if m else -1

						UpperCamelCase__   =    [re.compile(snake_case ,  re.MULTILINE       ) for pattern in truncate_before_pattern]

						UpperCamelCase__   =    list(re.finditer("^print" ,  snake_case ,  re.MULTILINE       )       )

						if len(snake_case       ) > 1:
									UpperCamelCase__   =    completion[: prints[1].start()]

						UpperCamelCase__   =    list(re.finditer("^def" ,  snake_case ,  re.MULTILINE       )       )

						if len(snake_case       ) > 1:
									UpperCamelCase__   =    completion[: defs[1].start()]

						UpperCamelCase__   =    0

						UpperCamelCase__   =    [
						    pos for pos in [find_re(snake_case ,  snake_case ,  snake_case       ) for terminal in terminals] if pos != -1
						]

						if len(snake_case       ) > 0:
									return completion[: min(snake_case       )]
						else:
									return completion


