




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


lowercase_		=							logging.getLogger(__name__)





class 		A__    (       __SCREAMING_SNAKE_CASE			):






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase=None    ,					lowerCamelCase=None						)				->       Optional[Any]:




						"""simple docstring"""






						if current_layer == 0:
											__magic_name__ :							List[str]		       =							self.embedding_hidden_mapping_in(lowerCamelCase						)
						else:
											__magic_name__ :							int		       =							hidden_states[0]

						__magic_name__ :							Union[str, Any]		       =							int(self.config.num_hidden_layers / self.config.num_hidden_groups						)

						# Index of the hidden group
						__magic_name__ :							List[Any]		       =							int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)						)

						__magic_name__ :							Optional[Any]		       =							self.albert_layer_groups[group_idx](
						    lowerCamelCase    ,					lowerCamelCase    ,					head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]    ,					)
						__magic_name__ :							Any		       =							layer_group_output[0]

						return (hidden_states,)





@add_start_docstrings(
    "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."		,					__SCREAMING_SNAKE_CASE		,					)
class 		A__    (       __SCREAMING_SNAKE_CASE			):






	def __init__(     self    ,					lowerCamelCase						)				->       Tuple:




						"""simple docstring"""






						super().__init__(lowerCamelCase						)

						__magic_name__ :							Dict		       =							AlbertTransformerWithPabee(lowerCamelCase						)

						self.init_weights()
						__magic_name__ :							List[str]		       =							0
						__magic_name__ :							Optional[int]		       =							0
						__magic_name__ :							Union[str, Any]		       =							0

						__magic_name__ :							List[str]		       =							0






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Dict:




						"""simple docstring"""






						__magic_name__ :							int		       =							threshold






	def 			lowercase				(     self    ,					lowerCamelCase						)				->       Optional[Any]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							patience






	def 			lowercase				(     self						)				->       Dict:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							0
						__magic_name__ :							Union[str, Any]		       =							0






	def 			lowercase				(     self						)				->       str:




						"""simple docstring"""






						__magic_name__ :							Union[str, Any]		       =							self.inference_layers_num / self.inference_instances_num
						__magic_name__ :							Union[str, Any]		       =							(
						    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
						    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
						)
						print(lowerCamelCase						)






	@add_start_docstrings_to_model_forward(lowerCamelCase						)
	def 			lowercase				(     self    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=False    ,					)				->       Tuple:




						"""simple docstring"""






						if input_ids is not None and inputs_embeds is not None:
											raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''						)
						elif input_ids is not None:
											__magic_name__ :							int		       =							input_ids.size()
						elif inputs_embeds is not None:
											__magic_name__ :							int		       =							inputs_embeds.size()[:-1]
						else:
											raise ValueError('''You have to specify either input_ids or inputs_embeds'''						)

						__magic_name__ :							Any		       =							input_ids.device if input_ids is not None else inputs_embeds.device

						if attention_mask is None:
											__magic_name__ :							Dict		       =							torch.ones(lowerCamelCase    ,					device=lowerCamelCase						)
						if token_type_ids is None:
											__magic_name__ :							str		       =							torch.zeros(lowerCamelCase    ,					dtype=torch.long    ,					device=lowerCamelCase						)

						__magic_name__ :							List[Any]		       =							attention_mask.unsqueeze(1						).unsqueeze(2						)
						__magic_name__ :							str		       =							extended_attention_mask.to(dtype=self.dtype						)  # fp16 compatibility
						__magic_name__ :							Union[str, Any]		       =							(1.0 - extended_attention_mask) * -1_0_0_0_0.0
						__magic_name__ :							Optional[Any]		       =							self.get_head_mask(lowerCamelCase    ,					self.config.num_hidden_layers						)

						__magic_name__ :							Dict		       =							self.embeddings(
						    lowerCamelCase    ,					position_ids=lowerCamelCase    ,					token_type_ids=lowerCamelCase    ,					inputs_embeds=lowerCamelCase						)
						__magic_name__ :							str		       =							embedding_output

						if self.training:
											__magic_name__ :							int		       =							[]
											for i in range(self.config.num_hidden_layers						):
																__magic_name__ :							List[Any]		       =							self.encoder.adaptive_forward(
																    lowerCamelCase    ,					current_layer=lowerCamelCase    ,					attention_mask=lowerCamelCase    ,					head_mask=lowerCamelCase    ,					)

																__magic_name__ :							str		       =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
																__magic_name__ :							List[str]		       =							output_layers[i](output_dropout(lowerCamelCase						)						)
																res.append(lowerCamelCase						)
						elif self.patience == 0:  # Use all layers for inference
											__magic_name__ :							Union[str, Any]		       =							self.encoder(lowerCamelCase    ,					lowerCamelCase    ,					head_mask=lowerCamelCase						)
											__magic_name__ :							Union[str, Any]		       =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
											__magic_name__ :							int		       =							[output_layers[self.config.num_hidden_layers - 1](lowerCamelCase						)]
						else:
											__magic_name__ :							List[Any]		       =							0
											__magic_name__ :							Tuple		       =							None
											__magic_name__ :							Tuple		       =							0
											for i in range(self.config.num_hidden_layers						):
																calculated_layer_num += 1
																__magic_name__ :							int		       =							self.encoder.adaptive_forward(
																    lowerCamelCase    ,					current_layer=lowerCamelCase    ,					attention_mask=lowerCamelCase    ,					head_mask=lowerCamelCase    ,					)

																__magic_name__ :							Any		       =							self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
																__magic_name__ :							List[str]		       =							output_layers[i](lowerCamelCase						)
																if regression:
																					__magic_name__ :							Optional[int]		       =							logits.detach()
																					if patient_result is not None:
																										__magic_name__ :							Any		       =							patient_result.detach()
																					if (patient_result is not None) and torch.abs(patient_result - labels						) < self.regression_threshold:
																										patient_counter += 1
																					else:
																										__magic_name__ :							Optional[int]		       =							0
																else:
																					__magic_name__ :							Optional[Any]		       =							logits.detach().argmax(dim=1						)
																					if patient_result is not None:
																										__magic_name__ :							Union[str, Any]		       =							patient_result.detach().argmax(dim=1						)
																					if (patient_result is not None) and torch.all(labels.eq(lowerCamelCase						)						):
																										patient_counter += 1
																					else:
																										__magic_name__ :							Tuple		       =							0

																__magic_name__ :							List[str]		       =							logits
																if patient_counter == self.patience:
																					break
											__magic_name__ :							List[str]		       =							[patient_result]
											self.inference_layers_num += calculated_layer_num
											self.inference_instances_num += 1

						return res





@add_start_docstrings(
    "Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "		,					__SCREAMING_SNAKE_CASE		,					)
class 		A__    (       __SCREAMING_SNAKE_CASE			):






	def __init__(     self    ,					lowerCamelCase						)				->       Any:




						"""simple docstring"""






						super().__init__(lowerCamelCase						)
						__magic_name__ :							List[Any]		       =							config.num_labels

						__magic_name__ :							str		       =							AlbertModelWithPabee(lowerCamelCase						)
						__magic_name__ :							List[Any]		       =							nn.Dropout(config.classifier_dropout_prob						)
						__magic_name__ :							Union[str, Any]		       =							nn.ModuleList(
						    [nn.Linear(config.hidden_size    ,					self.config.num_labels						) for _ in range(config.num_hidden_layers						)]						)

						self.init_weights()






	@add_start_docstrings_to_model_forward(lowerCamelCase						)
	def 			lowercase				(     self    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					lowerCamelCase=None    ,					)				->       Union[str, Any]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							self.albert(
						    input_ids=lowerCamelCase    ,					attention_mask=lowerCamelCase    ,					token_type_ids=lowerCamelCase    ,					position_ids=lowerCamelCase    ,					head_mask=lowerCamelCase    ,					inputs_embeds=lowerCamelCase    ,					output_dropout=self.dropout    ,					output_layers=self.classifiers    ,					regression=self.num_labels == 1    ,					)

						__magic_name__ :							Dict		       =							(logits[-1],)

						if labels is not None:
											__magic_name__ :							Optional[Any]		       =							None
											__magic_name__ :							Tuple		       =							0
											for ix, logits_item in enumerate(lowerCamelCase						):
																if self.num_labels == 1:
																					#  We are doing regression
																					__magic_name__ :							str		       =							MSELoss()
																					__magic_name__ :							Dict		       =							loss_fct(logits_item.view(-1						)    ,					labels.view(-1						)						)
																else:
																					__magic_name__ :							str		       =							CrossEntropyLoss()
																					__magic_name__ :							int		       =							loss_fct(logits_item.view(-1    ,					self.num_labels						)    ,					labels.view(-1						)						)
																if total_loss is None:
																					__magic_name__ :							Dict		       =							loss
																else:
																					total_loss += loss * (ix + 1)
																total_weights += ix + 1
											__magic_name__ :							Dict		       =							(total_loss / total_weights,) + outputs

						return outputs









import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
				import torch

				from transformers import CamembertModel





@require_torch
@require_sentencepiece
@require_tokenizers
class 		A__    (       unittest.TestCase			):






	@slow
	def 			lowercase				(     self						)				->       int:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							CamembertModel.from_pretrained('''camembert-base'''						)
						model.to(lowerCamelCase						)

						__magic_name__ :							Dict		       =							torch.tensor(
						    [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]]    ,					device=lowerCamelCase    ,					dtype=torch.long    ,					)  # J'aime le camembert !
						with torch.no_grad():
											__magic_name__ :							Tuple		       =							model(lowerCamelCase						)['''last_hidden_state''']
						__magic_name__ :							int		       =							torch.Size((1, 10, 768)						)
						self.assertEqual(output.shape    ,					lowerCamelCase						)
						# compare the actual values for a slice.
						__magic_name__ :							Tuple		       =							torch.tensor(
						    [[[-0.0_2_5_4, 0.0_2_3_5, 0.1_0_2_7], [0.0_6_0_6, -0.1_8_1_1, -0.0_4_1_8], [-0.1_5_6_1, -0.1_1_2_7, 0.2_6_8_7]]]    ,					device=lowerCamelCase    ,					dtype=torch.float    ,					)
						# camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
						# camembert.eval()
						# expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()

						self.assertTrue(torch.allclose(output[:, :3, :3]    ,					lowerCamelCase    ,					atol=1e-4						)						)



