



import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


UpperCAmelCase				:							List[Any]								=logging.getLogger(__name__)


class  _lowercase				(a_ ):
				'''simple docstring'''







				def  _lowerCamelCase   (	self       ,	snake_case__       ,	snake_case__       ,	snake_case__=None       ,	snake_case__=None      ):

									'''simple docstring'''


									if current_layer == 0:
														UpperCamelCase_											=				self.embedding_hidden_mapping_in(snake_case__      )
									else:
														UpperCamelCase_											=				hidden_states[0]

									UpperCamelCase_											=				int(self.config.num_hidden_layers / self.config.num_hidden_groups      )

									# Index of the hidden group
									UpperCamelCase_											=				int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)      )

									UpperCamelCase_											=				self.albert_layer_groups[group_idx](
									    snake_case__       ,	snake_case__       ,	head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]       ,	)
									UpperCamelCase_											=				layer_group_output[0]

									return (hidden_states,)




@add_start_docstrings(
    """The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.""" ,				a_ ,				)
class  _lowercase				(a_ ):
				'''simple docstring'''







				def __init__(	self       ,	snake_case__      ):

									'''simple docstring'''


									super().__init__(snake_case__      )

									UpperCamelCase_											=				AlbertTransformerWithPabee(snake_case__      )

									self.init_weights()
									UpperCamelCase_											=				0
									UpperCamelCase_											=				0
									UpperCamelCase_											=				0

									UpperCamelCase_											=				0

				def  _lowerCamelCase   (	self       ,	snake_case__      ):

									'''simple docstring'''


									UpperCamelCase_											=				threshold

				def  _lowerCamelCase   (	self       ,	snake_case__      ):

									'''simple docstring'''


									UpperCamelCase_											=				patience

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				0
									UpperCamelCase_											=				0

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.inference_layers_num / self.inference_instances_num
									UpperCamelCase_											=				(
									    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
									    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
									)
									print(snake_case__      )

				@add_start_docstrings_to_model_forward(snake_case__      )
				def  _lowerCamelCase   (	self       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=False       ,	):

									'''simple docstring'''


									if input_ids is not None and inputs_embeds is not None:
														raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"      )
									elif input_ids is not None:
														UpperCamelCase_											=				input_ids.size()
									elif inputs_embeds is not None:
														UpperCamelCase_											=				inputs_embeds.size()[:-1]
									else:
														raise ValueError("You have to specify either input_ids or inputs_embeds"      )

									UpperCamelCase_											=				input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
														UpperCamelCase_											=				torch.ones(snake_case__       ,	device=snake_case__      )
									if token_type_ids is None:
														UpperCamelCase_											=				torch.zeros(snake_case__       ,	dtype=torch.long       ,	device=snake_case__      )

									UpperCamelCase_											=				attention_mask.unsqueeze(1      ).unsqueeze(2      )
									UpperCamelCase_											=				extended_attention_mask.to(dtype=self.dtype      )  # fp16 compatibility
									UpperCamelCase_											=				(1.0 - extended_attention_mask) * -10_000.0
									UpperCamelCase_											=				self.get_head_mask(snake_case__       ,	self.config.num_hidden_layers      )

									UpperCamelCase_											=				self.embeddings(
									    snake_case__       ,	position_ids=snake_case__       ,	token_type_ids=snake_case__       ,	inputs_embeds=snake_case__      )
									UpperCamelCase_											=				embedding_output

									if self.training:
														UpperCamelCase_											=				[]
														for i in range(self.config.num_hidden_layers      ):
																			UpperCamelCase_											=				self.encoder.adaptive_forward(
																			    snake_case__       ,	current_layer=snake_case__       ,	attention_mask=snake_case__       ,	head_mask=snake_case__       ,	)

																			UpperCamelCase_											=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
																			UpperCamelCase_											=				output_layers[i](output_dropout(snake_case__      )      )
																			res.append(snake_case__      )
									elif self.patience == 0:  # Use all layers for inference
														UpperCamelCase_											=				self.encoder(snake_case__       ,	snake_case__       ,	head_mask=snake_case__      )
														UpperCamelCase_											=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
														UpperCamelCase_											=				[output_layers[self.config.num_hidden_layers - 1](snake_case__      )]
									else:
														UpperCamelCase_											=				0
														UpperCamelCase_											=				None
														UpperCamelCase_											=				0
														for i in range(self.config.num_hidden_layers      ):
																			calculated_layer_num += 1
																			UpperCamelCase_											=				self.encoder.adaptive_forward(
																			    snake_case__       ,	current_layer=snake_case__       ,	attention_mask=snake_case__       ,	head_mask=snake_case__       ,	)

																			UpperCamelCase_											=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]      )      )
																			UpperCamelCase_											=				output_layers[i](snake_case__      )
																			if regression:
																								UpperCamelCase_											=				logits.detach()
																								if patient_result is not None:
																													UpperCamelCase_											=				patient_result.detach()
																								if (patient_result is not None) and torch.abs(patient_result - labels      ) < self.regression_threshold:
																													patient_counter += 1
																								else:
																													UpperCamelCase_											=				0
																			else:
																								UpperCamelCase_											=				logits.detach().argmax(dim=1      )
																								if patient_result is not None:
																													UpperCamelCase_											=				patient_result.detach().argmax(dim=1      )
																								if (patient_result is not None) and torch.all(labels.eq(snake_case__      )      ):
																													patient_counter += 1
																								else:
																													UpperCamelCase_											=				0

																			UpperCamelCase_											=				logits
																			if patient_counter == self.patience:
																								break
														UpperCamelCase_											=				[patient_result]
														self.inference_layers_num += calculated_layer_num
														self.inference_instances_num += 1

									return res




@add_start_docstrings(
    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """ ,				a_ ,				)
class  _lowercase				(a_ ):
				'''simple docstring'''







				def __init__(	self       ,	snake_case__      ):

									'''simple docstring'''


									super().__init__(snake_case__      )
									UpperCamelCase_											=				config.num_labels

									UpperCamelCase_											=				AlbertModelWithPabee(snake_case__      )
									UpperCamelCase_											=				nn.Dropout(config.classifier_dropout_prob      )
									UpperCamelCase_											=				nn.ModuleList(
									    [nn.Linear(config.hidden_size       ,	self.config.num_labels      ) for _ in range(config.num_hidden_layers      )]      )

									self.init_weights()

				@add_start_docstrings_to_model_forward(snake_case__      )
				def  _lowerCamelCase   (	self       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	snake_case__=None       ,	):

									'''simple docstring'''


									UpperCamelCase_											=				self.albert(
									    input_ids=snake_case__       ,	attention_mask=snake_case__       ,	token_type_ids=snake_case__       ,	position_ids=snake_case__       ,	head_mask=snake_case__       ,	inputs_embeds=snake_case__       ,	output_dropout=self.dropout       ,	output_layers=self.classifiers       ,	regression=self.num_labels == 1       ,	)

									UpperCamelCase_											=				(logits[-1],)

									if labels is not None:
														UpperCamelCase_											=				None
														UpperCamelCase_											=				0
														for ix, logits_item in enumerate(snake_case__      ):
																			if self.num_labels == 1:
																								#  We are doing regression
																								UpperCamelCase_											=				MSELoss()
																								UpperCamelCase_											=				loss_fct(logits_item.view(-1      )       ,	labels.view(-1      )      )
																			else:
																								UpperCamelCase_											=				CrossEntropyLoss()
																								UpperCamelCase_											=				loss_fct(logits_item.view(-1       ,	self.num_labels      )       ,	labels.view(-1      )      )
																			if total_loss is None:
																								UpperCamelCase_											=				loss
																			else:
																								total_loss += loss * (ix + 1)
																			total_weights += ix + 1
														UpperCamelCase_											=				(total_loss / total_weights,) + outputs

									return outputs





from typing import List, Optional, Tuple, Union

from . import logging
from .import_utils import is_torch_available, is_torch_version


if is_torch_available():
	import torch

UpperCAmelCase				:							Any								=logging.get_logger(__name__)  # pylint: disable=invalid-name

try:
	from torch._dynamo import allow_in_graph as maybe_allow_in_graph
except (ImportError, ModuleNotFoundError):



	def 				_lowerCAmelCase      (cls):
						return cls



def 				_lowerCAmelCase      (_lowerCAmelCase				,		_lowerCAmelCase = None				,		_lowerCAmelCase = None				,		_lowerCAmelCase = None				,		_lowerCAmelCase = None				,		):
					UpperCamelCase_											=				device
					UpperCamelCase_											=				shape[0]

					UpperCamelCase_											=				layout or torch.strided
					UpperCamelCase_											=				device or torch.device("cpu")

					if generator is not None:
										UpperCamelCase_											=				generator.device.type if not isinstance(_lowerCAmelCase				,		_lowerCAmelCase) else generator[0].device.type
										if gen_device_type != device.type and gen_device_type == "cpu":
															UpperCamelCase_											=				"cpu"
															if device != "mps":
																				logger.info(
																				    f"""The passed generator was created on 'cpu' even though a tensor on {device} was expected."""
																				    f""" Tensors will be created on 'cpu' and then moved to {device}. Note that one can probably"""
																				    f""" slighly speed up this function by passing a generator that was created on the {device} device.""")
										elif gen_device_type != device.type and gen_device_type == "cuda":
															raise ValueError(f"""Cannot generate a {device} tensor from a generator of type {gen_device_type}.""")

					if isinstance(_lowerCAmelCase				,		_lowerCAmelCase):
										UpperCamelCase_											=				(1,) + shape[1:]
										UpperCamelCase_											=				[
										    torch.randn(_lowerCAmelCase				,		generator=generator[i]				,		device=_lowerCAmelCase				,		dtype=_lowerCAmelCase				,		layout=_lowerCAmelCase)
										    for i in range(_lowerCAmelCase)
										]
										UpperCamelCase_											=				torch.cat(_lowerCAmelCase				,		dim=0).to(_lowerCAmelCase)
					else:
										UpperCamelCase_											=				torch.randn(_lowerCAmelCase				,		generator=_lowerCAmelCase				,		device=_lowerCAmelCase				,		dtype=_lowerCAmelCase				,		layout=_lowerCAmelCase).to(_lowerCAmelCase)

					return latents






def 				_lowerCAmelCase      (_lowerCAmelCase):
					if is_torch_version("<"				,		"2.0.0") or not hasattr(_lowerCAmelCase				,		"_dynamo"):
										return False
					return isinstance(_lowerCAmelCase				,		torch._dynamo.eval_frame.OptimizedModule)
