




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_lowerCamelCase					:     List[Any]				       =    logging.getLogger(__name__)

class     lowercase (      a_):







				def    a_					(							self				:   Optional[int] ,       _lowerCamelCase				:   int ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   List[Any]=None ,       _lowerCamelCase				:   Dict=None    ):



					"""simple docstring"""

					if current_layer == 0:
						A_					:			Dict						=						self.embedding_hidden_mapping_in(snake_case__    )
					else:
						A_					:			int						=						hidden_states[0]

					A_					:			Optional[Any]						=						int(self.config.num_hidden_layers / self.config.num_hidden_groups    )

					# Index of the hidden group
					A_					:			Union[str, Any]						=						int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)    )

					A_					:			Union[str, Any]						=						self.albert_layer_groups[group_idx](
					    snake_case__ ,       snake_case__ ,       head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group] ,       )
					A_					:			int						=						layer_group_output[0]

					return (hidden_states,)

@add_start_docstrings(
    """The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."""       ,		a_       ,		)
class     lowercase (      a_):







				def __init__(							self				:   Optional[Any] ,       _lowerCamelCase				:   Union[str, Any]    ):



					"""simple docstring"""

					super().__init__(snake_case__    )

					A_					:			Optional[int]						=						AlbertTransformerWithPabee(snake_case__    )

					self.init_weights()
					A_					:			List[Any]						=						0
					A_					:			List[str]						=						0
					A_					:			Tuple						=						0

					A_					:			List[str]						=						0







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   Dict    ):



					"""simple docstring"""

					A_					:			Union[str, Any]						=						threshold







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   Dict    ):



					"""simple docstring"""

					A_					:			str						=						patience







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			str						=						0
					A_					:			List[Any]						=						0







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.inference_layers_num / self.inference_instances_num
					A_					:			List[Any]						=						(
					    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
					    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
					)
					print(snake_case__    )







				@add_start_docstrings_to_model_forward(snake_case__    )
				def    a_					(							self				:   Dict ,       _lowerCamelCase				:   Tuple=None ,       _lowerCamelCase				:   Tuple=None ,       _lowerCamelCase				:   Union[str, Any]=None ,       _lowerCamelCase				:   Any=None ,       _lowerCamelCase				:   Optional[int]=None ,       _lowerCamelCase				:   Union[str, Any]=None ,       _lowerCamelCase				:   Optional[int]=None ,       _lowerCamelCase				:   str=None ,       _lowerCamelCase				:   int=False ,       ):



					"""simple docstring"""

					if input_ids is not None and inputs_embeds is not None:
						raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''    )
					elif input_ids is not None:
						A_					:			Any						=						input_ids.size()
					elif inputs_embeds is not None:
						A_					:			Dict						=						inputs_embeds.size()[:-1]
					else:
						raise ValueError('''You have to specify either input_ids or inputs_embeds'''    )

					A_					:			Optional[int]						=						input_ids.device if input_ids is not None else inputs_embeds.device

					if attention_mask is None:
						A_					:			Tuple						=						torch.ones(snake_case__ ,       device=snake_case__    )
					if token_type_ids is None:
						A_					:			List[Any]						=						torch.zeros(snake_case__ ,       dtype=torch.long ,       device=snake_case__    )

					A_					:			Tuple						=						attention_mask.unsqueeze(1    ).unsqueeze(2    )
					A_					:			Any						=						extended_attention_mask.to(dtype=self.dtype    )  # fp16 compatibility
					A_					:			Optional[Any]						=						(1.0 - extended_attention_mask) * -1_00_00.0
					A_					:			List[str]						=						self.get_head_mask(snake_case__ ,       self.config.num_hidden_layers    )

					A_					:			List[str]						=						self.embeddings(
					    snake_case__ ,       position_ids=snake_case__ ,       token_type_ids=snake_case__ ,       inputs_embeds=snake_case__    )
					A_					:			str						=						embedding_output

					if self.training:
						A_					:			str						=						[]
						for i in range(self.config.num_hidden_layers    ):
							A_					:			List[str]						=						self.encoder.adaptive_forward(
							    snake_case__ ,       current_layer=snake_case__ ,       attention_mask=snake_case__ ,       head_mask=snake_case__ ,       )

							A_					:			Dict						=						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
							A_					:			Dict						=						output_layers[i](output_dropout(snake_case__    )    )
							res.append(snake_case__    )
					elif self.patience == 0:  # Use all layers for inference
						A_					:			Dict						=						self.encoder(snake_case__ ,       snake_case__ ,       head_mask=snake_case__    )
						A_					:			List[Any]						=						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
						A_					:			Optional[int]						=						[output_layers[self.config.num_hidden_layers - 1](snake_case__    )]
					else:
						A_					:			str						=						0
						A_					:			str						=						None
						A_					:			int						=						0
						for i in range(self.config.num_hidden_layers    ):
							calculated_layer_num += 1
							A_					:			Union[str, Any]						=						self.encoder.adaptive_forward(
							    snake_case__ ,       current_layer=snake_case__ ,       attention_mask=snake_case__ ,       head_mask=snake_case__ ,       )

							A_					:			Any						=						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
							A_					:			Optional[Any]						=						output_layers[i](snake_case__    )
							if regression:
								A_					:			Tuple						=						logits.detach()
								if patient_result is not None:
									A_					:			Optional[int]						=						patient_result.detach()
								if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
									patient_counter += 1
								else:
									A_					:			Tuple						=						0
							else:
								A_					:			str						=						logits.detach().argmax(dim=1    )
								if patient_result is not None:
									A_					:			Dict						=						patient_result.detach().argmax(dim=1    )
								if (patient_result is not None) and torch.all(labels.eq(snake_case__    )    ):
									patient_counter += 1
								else:
									A_					:			Optional[Any]						=						0

							A_					:			Union[str, Any]						=						logits
							if patient_counter == self.patience:
								break
						A_					:			Tuple						=						[patient_result]
						self.inference_layers_num += calculated_layer_num
						self.inference_instances_num += 1

					return res

@add_start_docstrings(
    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """       ,		a_       ,		)
class     lowercase (      a_):







				def __init__(							self				:   Any ,       _lowerCamelCase				:   Optional[int]    ):



					"""simple docstring"""

					super().__init__(snake_case__    )
					A_					:			List[Any]						=						config.num_labels

					A_					:			Union[str, Any]						=						AlbertModelWithPabee(snake_case__    )
					A_					:			Dict						=						nn.Dropout(config.classifier_dropout_prob    )
					A_					:			List[Any]						=						nn.ModuleList(
					    [nn.Linear(config.hidden_size ,       self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )

					self.init_weights()







				@add_start_docstrings_to_model_forward(snake_case__    )
				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   Optional[Any]=None ,       _lowerCamelCase				:   List[str]=None ,       _lowerCamelCase				:   Tuple=None ,       _lowerCamelCase				:   str=None ,       _lowerCamelCase				:   Optional[Any]=None ,       _lowerCamelCase				:   Optional[Any]=None ,       _lowerCamelCase				:   str=None ,       ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.albert(
					    input_ids=snake_case__ ,       attention_mask=snake_case__ ,       token_type_ids=snake_case__ ,       position_ids=snake_case__ ,       head_mask=snake_case__ ,       inputs_embeds=snake_case__ ,       output_dropout=self.dropout ,       output_layers=self.classifiers ,       regression=self.num_labels == 1 ,       )

					A_					:			List[str]						=						(logits[-1],)

					if labels is not None:
						A_					:			Union[str, Any]						=						None
						A_					:			int						=						0
						for ix, logits_item in enumerate(snake_case__    ):
							if self.num_labels == 1:
								#  We are doing regression
								A_					:			str						=						MSELoss()
								A_					:			Dict						=						loss_fct(logits_item.view(-1    ) ,       labels.view(-1    )    )
							else:
								A_					:			int						=						CrossEntropyLoss()
								A_					:			Optional[Any]						=						loss_fct(logits_item.view(-1 ,       self.num_labels    ) ,       labels.view(-1    )    )
							if total_loss is None:
								A_					:			int						=						loss
							else:
								total_loss += loss * (ix + 1)
							total_weights += ix + 1
						A_					:			int						=						(total_loss / total_weights,) + outputs

					return outputs











"""simple docstring"""

def 						lowercase_ (						_UpperCAmelCase		):




	"""simple docstring"""






	A_					:			Optional[int]						=						9

	while num > 0:
		if num % 10 != digit:
			return False
		num //= 100
		digit -= 1

	return True

def 						lowercase_ (						):




	"""simple docstring"""






	A_					:			Dict						=						1_3890_2663

	while not is_square_form(num * num		):
		if num % 10 == 3:
			num -= 6  # (3 - 6) % 10 = 7
		else:
			num -= 4  # (7 - 4) % 10 = 3

	return num * 10


if __name__ == "__main__":
		print(f'{solution() = }')





