"""simple docstring"""


import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


a							=logging.getLogger(__name__)






class        __UpperCAmelCase						(     __lowerCAmelCase     ):

				def 		_a     (				self     ,			_lowerCamelCase     ,			_lowerCamelCase     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None				):
										if current_layer == 0:
																lowerCamelCase__ =self.embedding_hidden_mapping_in(_lowerCamelCase				)
										else:
																lowerCamelCase__ =hidden_states[0]

										lowerCamelCase__ =int(self.config.num_hidden_layers / self.config.num_hidden_groups				)

										# Index of the hidden group
										lowerCamelCase__ =int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)				)

										lowerCamelCase__ =self.albert_layer_groups[group_idx](
										    _lowerCamelCase     ,			_lowerCamelCase     ,			head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]     ,			)
										lowerCamelCase__ =layer_group_output[0]

										return (hidden_states,)






@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''				,				__lowerCAmelCase				,				)
class        __UpperCAmelCase						(     __lowerCAmelCase     ):

				def __init__(				self     ,			_lowerCamelCase				):
										super().__init__(_lowerCamelCase				)

										lowerCamelCase__ =AlbertTransformerWithPabee(_lowerCamelCase				)

										self.init_weights()
										lowerCamelCase__ =0
										lowerCamelCase__ =0
										lowerCamelCase__ =0

										lowerCamelCase__ =0

				def 		_a     (				self     ,			_lowerCamelCase				):
										lowerCamelCase__ =threshold

				def 		_a     (				self     ,			_lowerCamelCase				):
										lowerCamelCase__ =patience

				def 		_a     (				self				):
										lowerCamelCase__ =0
										lowerCamelCase__ =0

				def 		_a     (				self				):
										lowerCamelCase__ =self.inference_layers_num / self.inference_instances_num
										lowerCamelCase__ =(
										    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
										    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
										)
										print(_lowerCamelCase				)






				@add_start_docstrings_to_model_forward(_lowerCamelCase				)
				def 		_a     (				self     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=False     ,			):

										if input_ids is not None and inputs_embeds is not None:
																raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"				)
										elif input_ids is not None:
																lowerCamelCase__ =input_ids.size()
										elif inputs_embeds is not None:
																lowerCamelCase__ =inputs_embeds.size()[:-1]
										else:
																raise ValueError("You have to specify either input_ids or inputs_embeds"				)

										lowerCamelCase__ =input_ids.device if input_ids is not None else inputs_embeds.device

										if attention_mask is None:
																lowerCamelCase__ =torch.ones(_lowerCamelCase     ,			device=_lowerCamelCase				)
										if token_type_ids is None:
																lowerCamelCase__ =torch.zeros(_lowerCamelCase     ,			dtype=torch.long     ,			device=_lowerCamelCase				)

										lowerCamelCase__ =attention_mask.unsqueeze(1				).unsqueeze(2				)
										lowerCamelCase__ =extended_attention_mask.to(dtype=self.dtype				)  # fp16 compatibility
										lowerCamelCase__ =(1.0 - extended_attention_mask) * -10000.0
										lowerCamelCase__ =self.get_head_mask(_lowerCamelCase     ,			self.config.num_hidden_layers				)

										lowerCamelCase__ =self.embeddings(
										    _lowerCamelCase     ,			position_ids=_lowerCamelCase     ,			token_type_ids=_lowerCamelCase     ,			inputs_embeds=_lowerCamelCase				)
										lowerCamelCase__ =embedding_output

										if self.training:
																lowerCamelCase__ =[]
																for i in range(self.config.num_hidden_layers				):
																						lowerCamelCase__ =self.encoder.adaptive_forward(
																						    _lowerCamelCase     ,			current_layer=_lowerCamelCase     ,			attention_mask=_lowerCamelCase     ,			head_mask=_lowerCamelCase     ,			)

																						lowerCamelCase__ =self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]				)				)
																						lowerCamelCase__ =output_layers[i](output_dropout(_lowerCamelCase				)				)
																						res.append(_lowerCamelCase				)
										elif self.patience == 0:  # Use all layers for inference
																lowerCamelCase__ =self.encoder(_lowerCamelCase     ,			_lowerCamelCase     ,			head_mask=_lowerCamelCase				)
																lowerCamelCase__ =self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]				)				)
																lowerCamelCase__ =[output_layers[self.config.num_hidden_layers - 1](_lowerCamelCase				)]
										else:
																lowerCamelCase__ =0
																lowerCamelCase__ =None
																lowerCamelCase__ =0
																for i in range(self.config.num_hidden_layers				):
																						calculated_layer_num += 1
																						lowerCamelCase__ =self.encoder.adaptive_forward(
																						    _lowerCamelCase     ,			current_layer=_lowerCamelCase     ,			attention_mask=_lowerCamelCase     ,			head_mask=_lowerCamelCase     ,			)

																						lowerCamelCase__ =self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]				)				)
																						lowerCamelCase__ =output_layers[i](_lowerCamelCase				)
																						if regression:
																												lowerCamelCase__ =logits.detach()
																												if patient_result is not None:
																																		lowerCamelCase__ =patient_result.detach()
																												if (patient_result is not None) and torch.abs(patient_result - labels				) < self.regression_threshold:
																																		patient_counter += 1
																												else:
																																		lowerCamelCase__ =0
																						else:
																												lowerCamelCase__ =logits.detach().argmax(dim=1				)
																												if patient_result is not None:
																																		lowerCamelCase__ =patient_result.detach().argmax(dim=1				)
																												if (patient_result is not None) and torch.all(labels.eq(_lowerCamelCase				)				):
																																		patient_counter += 1
																												else:
																																		lowerCamelCase__ =0

																						lowerCamelCase__ =logits
																						if patient_counter == self.patience:
																												break
																lowerCamelCase__ =[patient_result]
																self.inference_layers_num += calculated_layer_num
																self.inference_instances_num += 1

										return res






@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''				,				__lowerCAmelCase				,				)
class        __UpperCAmelCase						(     __lowerCAmelCase     ):

				def __init__(				self     ,			_lowerCamelCase				):
										super().__init__(_lowerCamelCase				)
										lowerCamelCase__ =config.num_labels

										lowerCamelCase__ =AlbertModelWithPabee(_lowerCamelCase				)
										lowerCamelCase__ =nn.Dropout(config.classifier_dropout_prob				)
										lowerCamelCase__ =nn.ModuleList(
										    [nn.Linear(config.hidden_size     ,			self.config.num_labels				) for _ in range(config.num_hidden_layers				)]				)

										self.init_weights()






				@add_start_docstrings_to_model_forward(_lowerCamelCase				)
				def 		_a     (				self     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			_lowerCamelCase=None     ,			):
										lowerCamelCase__ =self.albert(
										    input_ids=_lowerCamelCase     ,			attention_mask=_lowerCamelCase     ,			token_type_ids=_lowerCamelCase     ,			position_ids=_lowerCamelCase     ,			head_mask=_lowerCamelCase     ,			inputs_embeds=_lowerCamelCase     ,			output_dropout=self.dropout     ,			output_layers=self.classifiers     ,			regression=self.num_labels == 1     ,			)

										lowerCamelCase__ =(logits[-1],)

										if labels is not None:
																lowerCamelCase__ =None
																lowerCamelCase__ =0
																for ix, logits_item in enumerate(_lowerCamelCase				):
																						if self.num_labels == 1:
																												#  We are doing regression
																												lowerCamelCase__ =MSELoss()
																												lowerCamelCase__ =loss_fct(logits_item.view(-1				)     ,			labels.view(-1				)				)
																						else:
																												lowerCamelCase__ =CrossEntropyLoss()
																												lowerCamelCase__ =loss_fct(logits_item.view(-1     ,			self.num_labels				)     ,			labels.view(-1				)				)
																						if total_loss is None:
																												lowerCamelCase__ =loss
																						else:
																												total_loss += loss * (ix + 1)
																						total_weights += ix + 1
																lowerCamelCase__ =(total_loss / total_weights,) + outputs

										return outputs



"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a							={
    'configuration_openai': ['OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'OpenAIGPTConfig'],
    'tokenization_openai': ['OpenAIGPTTokenizer'],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a							=['OpenAIGPTTokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a							=[
	    'OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'OpenAIGPTDoubleHeadsModel',
	    'OpenAIGPTForSequenceClassification',
	    'OpenAIGPTLMHeadModel',
	    'OpenAIGPTModel',
	    'OpenAIGPTPreTrainedModel',
	    'load_tf_weights_in_openai_gpt',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a							=[
	    'TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFOpenAIGPTDoubleHeadsModel',
	    'TFOpenAIGPTForSequenceClassification',
	    'TFOpenAIGPTLMHeadModel',
	    'TFOpenAIGPTMainLayer',
	    'TFOpenAIGPTModel',
	    'TFOpenAIGPTPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
	from .tokenization_openai import OpenAIGPTTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_openai_fast import OpenAIGPTTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_openai import (
		    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    OpenAIGPTDoubleHeadsModel,
		    OpenAIGPTForSequenceClassification,
		    OpenAIGPTLMHeadModel,
		    OpenAIGPTModel,
		    OpenAIGPTPreTrainedModel,
		    load_tf_weights_in_openai_gpt,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_openai import (
		    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFOpenAIGPTDoubleHeadsModel,
		    TFOpenAIGPTForSequenceClassification,
		    TFOpenAIGPTLMHeadModel,
		    TFOpenAIGPTMainLayer,
		    TFOpenAIGPTModel,
		    TFOpenAIGPTPreTrainedModel,
		)

else:
	import sys

	a							=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


