



import numpy as np
from matplotlib import pyplot as plt
def 			UpperCamelCase__						(    A__   )       -> List[Any]:
				return 1 / (1 + np.exp(-1 * x   ))





class 			__snake_case  :




					def __init__(       self   ,	__UpperCamelCase   ,	__UpperCamelCase=None   ,	__UpperCamelCase=None   ,	__UpperCamelCase=False		)					->  List[Any]:



									'''simple docstring'''







									snake_case__ :     Any 							=				units
									snake_case__ :     Dict 							=				None
									snake_case__ :     int 							=				None
									snake_case__ :     Dict 							=				activation
									if learning_rate is None:
													snake_case__ :     List[str] 							=				0.3
									snake_case__ :     Tuple 							=				learning_rate
									snake_case__ :     List[Any] 							=				is_input_layer




					def       __a      (       self   ,	__UpperCamelCase		)					->  Any:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				np.asmatrix(np.random.normal(0   ,	0.5   ,	(self.units, back_units)		)		)
									snake_case__ :     List[str] 							=				np.asmatrix(np.random.normal(0   ,	0.5   ,	self.units		)		).T
									if self.activation is None:
													snake_case__ :     Optional[int] 							=				sigmoid




					def       __a      (       self		)					->  List[Any]:



									'''simple docstring'''







									if self.activation == sigmoid:
													snake_case__ :     Tuple 							=				np.dot(self.output   ,	(1 - self.output).T		)
													snake_case__ :     Union[str, Any] 							=				np.diag(np.diag(lowercase__		)		)
									else:
													snake_case__ :     Dict 							=				1
									return gradient_activation




					def       __a      (       self   ,	__UpperCamelCase		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     str 							=				xdata
									if self.is_input_layer:
													# input layer
													snake_case__ :     str 							=				xdata
													snake_case__ :     Union[str, Any] 							=				xdata
													return xdata
									else:
													snake_case__ :     Optional[Any] 							=				np.dot(self.weight   ,	self.xdata		) - self.bias
													snake_case__ :     List[Any] 							=				self.activation(self.wx_plus_b		)
													return self.output




					def       __a      (       self   ,	__UpperCamelCase		)					->  Union[str, Any]:



									'''simple docstring'''







									snake_case__ :     str 							=				self.cal_gradient()  # i * i 维
									snake_case__ :     List[Any] 							=				np.asmatrix(np.dot(gradient.T   ,	lowercase__		)		)

									snake_case__ :     List[str] 							=				np.asmatrix(self.xdata		)
									snake_case__ :     List[Any] 							=				-1
									snake_case__ :     Dict 							=				self.weight

									snake_case__ :     int 							=				np.dot(gradient.T   ,	self._gradient_weight.T		)
									snake_case__ :     str 							=				gradient * self._gradient_bias
									snake_case__ :     Tuple 							=				np.dot(lowercase__   ,	self._gradient_x		).T
									# upgrade: the Negative gradient direction
									snake_case__ :     List[Any] 							=				self.weight - self.learn_rate * self.gradient_weight
									snake_case__ :     List[str] 							=				self.bias - self.learn_rate * self.gradient_bias.T
									# updates the weights and bias according to learning rate (0.3 if undefined)
									return self.gradient





class 			__snake_case  :




					def __init__(       self		)					->  int:



									'''simple docstring'''







									snake_case__ :     Optional[int] 							=				[]
									snake_case__ :     Any 							=				[]
									snake_case__ :     Dict 							=				plt.figure()
									snake_case__ :     int 							=				self.fig_loss.add_subplot(1   ,	1   ,	1		)




					def       __a      (       self   ,	__UpperCamelCase		)					->  int:



									'''simple docstring'''







									self.layers.append(lowercase__		)




					def       __a      (       self		)					->  str:



									'''simple docstring'''







									for i, layer in enumerate(self.layers[:]		):
													if i < 1:
																	snake_case__ :     Optional[int] 							=				True
													else:
																	layer.initializer(self.layers[i - 1].units		)




					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									for i, layer in enumerate(self.layers[:]		):
													print(F"""------- layer {i} -------"""		)
													print('weight.shape '   ,	np.shape(layer.weight		)		)
													print('bias.shape '   ,	np.shape(layer.bias		)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase		)					->  str:



									'''simple docstring'''







									snake_case__ :     Optional[Any] 							=				train_round
									snake_case__ :     List[str] 							=				accuracy

									self.ax_loss.hlines(self.accuracy   ,	0   ,	self.train_round * 1.1		)

									snake_case__ :     Dict 							=				np.shape(lowercase__		)
									for _ in range(lowercase__		):
													snake_case__ :     Dict 							=				0
													for row in range(x_shape[0]		):
																	snake_case__ :     Dict 							=				np.asmatrix(xdata[row, :]		).T
																	snake_case__ :     List[Any] 							=				np.asmatrix(ydata[row, :]		).T

																	# forward propagation
																	for layer in self.layers:
																					snake_case__ :     Union[str, Any] 							=				layer.forward_propagation(_xdata		)

																	snake_case__ :     Dict 							=				self.cal_loss(_ydata   ,	_xdata		)
																	snake_case__ :     Optional[Any] 							=				all_loss + loss

																	# back propagation: the input_layer does not upgrade
																	for layer in self.layers[:0:-1]:
																					snake_case__ :     Union[str, Any] 							=				layer.back_propagation(lowercase__		)

													snake_case__ :     Tuple 							=				all_loss / x_shape[0]
													self.train_mse.append(lowercase__		)

													self.plot_loss()

													if mse < self.accuracy:
																	print('----达到精度----'		)
																	return mse
									return None




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase		)					->  Optional[int]:



									'''simple docstring'''







									snake_case__ :     Optional[int] 							=				np.sum(np.power((ydata - ydata_)   ,	2		)		)
									snake_case__ :     List[str] 							=				2 * (ydata_ - ydata)
									# vector (shape is the same as _ydata.shape)
									return self.loss, self.loss_gradient




					def       __a      (       self		)					->  Optional[int]:



									'''simple docstring'''







									if self.ax_loss.lines:
													self.ax_loss.lines.remove(self.ax_loss.lines[0]		)
									self.ax_loss.plot(self.train_mse   ,	'r-'		)
									plt.ion()
									plt.xlabel('step'		)
									plt.ylabel('loss'		)
									plt.show()
									plt.pause(0.1		)




def 			UpperCamelCase__						(    )       -> Optional[int]:
				snake_case__ :     Optional[Any] 							=				np.random.randn(10      ,			10   )
				snake_case__ :     Any 							=				np.asarray(
				    [
				        [0.8, 0.4],
				        [0.4, 0.3],
				        [0.3_4, 0.4_5],
				        [0.6_7, 0.3_2],
				        [0.8_8, 0.6_7],
				        [0.7_8, 0.7_7],
				        [0.5_5, 0.6_6],
				        [0.5_5, 0.4_3],
				        [0.5_4, 0.1],
				        [0.1, 0.5],
				    ]   )
				snake_case__ :     Union[str, Any] 							=				BPNN()
				for i in (10, 20, 30, 2):
								model.add_layer(DenseLayer(SCREAMING_SNAKE_CASE_   )   )
				model.build()
				model.summary()
				model.train(xdata=SCREAMING_SNAKE_CASE_      ,			ydata=SCREAMING_SNAKE_CASE_      ,			train_round=100      ,			accuracy=0.0_1   )


if __name__ == "__main__":
				example()



from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers.models.openai.modeling_tf_openai import (
				    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFOpenAIGPTDoubleHeadsModel,
				    TFOpenAIGPTForSequenceClassification,
				    TFOpenAIGPTLMHeadModel,
				    TFOpenAIGPTModel,
				)





class 			__snake_case  :




					def __init__(       self   ,	__UpperCamelCase   ,	)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     List[str] 							=				parent
									snake_case__ :     Optional[int] 							=				13
									snake_case__ :     Tuple 							=				7
									snake_case__ :     Optional[int] 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     Union[str, Any] 							=				True
									snake_case__ :     Optional[Any] 							=				99
									snake_case__ :     Dict 							=				32
									snake_case__ :     Union[str, Any] 							=				2
									snake_case__ :     Tuple 							=				4
									snake_case__ :     List[str] 							=				37
									snake_case__ :     Optional[int] 							=				'gelu'
									snake_case__ :     List[str] 							=				0.1
									snake_case__ :     Union[str, Any] 							=				0.1
									snake_case__ :     str 							=				512
									snake_case__ :     str 							=				16
									snake_case__ :     Dict 							=				2
									snake_case__ :     List[Any] 							=				0.0_2
									snake_case__ :     Union[str, Any] 							=				3
									snake_case__ :     Optional[Any] 							=				4
									snake_case__ :     List[Any] 							=				None
									snake_case__ :     Union[str, Any] 							=				self.vocab_size - 1




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     Any 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.vocab_size		)

									snake_case__ :     List[Any] 							=				None
									if self.use_input_mask:
													snake_case__ :     str 							=				random_attention_mask([self.batch_size, self.seq_length]		)

									snake_case__ :     List[str] 							=				None
									if self.use_token_type_ids:
													snake_case__ :     List[Any] 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.type_vocab_size		)

									snake_case__ :     Dict 							=				None
									if self.use_mc_token_ids:
													snake_case__ :     Any 							=				ids_tensor([self.batch_size, self.num_choices]   ,	self.seq_length		)

									snake_case__ :     List[Any] 							=				None
									snake_case__ :     Dict 							=				None
									snake_case__ :     str 							=				None
									if self.use_labels:
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size]   ,	self.type_sequence_label_size		)
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.num_labels		)
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size]   ,	self.num_choices		)

									snake_case__ :     str 							=				OpenAIGPTConfig(
									    vocab_size=self.vocab_size   ,	n_embd=self.hidden_size   ,	n_layer=self.num_hidden_layers   ,	n_head=self.num_attention_heads   ,	n_positions=self.max_position_embeddings   ,	pad_token_id=self.pad_token_id   ,	)

									snake_case__ :     Union[str, Any] 							=				ids_tensor([self.num_hidden_layers, self.num_attention_heads]   ,	2		)

									return (
									    config,
									    input_ids,
									    input_mask,
									    head_mask,
									    token_type_ids,
									    mc_token_ids,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  List[str]:



									'''simple docstring'''







									snake_case__ :     List[Any] 							=				TFOpenAIGPTModel(config=__UpperCamelCase		)
									snake_case__ :     str 							=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
									snake_case__ :     Dict 							=				model(__UpperCamelCase		)

									snake_case__ :     Tuple 							=				[input_ids, input_mask]
									snake_case__ :     List[str] 							=				model(__UpperCamelCase		)

									snake_case__ :     Tuple 							=				model(__UpperCamelCase		)

									self.parent.assertEqual(result.last_hidden_state.shape   ,	(self.batch_size, self.seq_length, self.hidden_size)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Optional[int]:



									'''simple docstring'''







									snake_case__ :     List[str] 							=				TFOpenAIGPTLMHeadModel(config=__UpperCamelCase		)
									snake_case__ :     Optional[int] 							=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
									snake_case__ :     List[str] 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(result.logits.shape   ,	(self.batch_size, self.seq_length, self.vocab_size)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     Dict 							=				TFOpenAIGPTDoubleHeadsModel(config=__UpperCamelCase		)

									snake_case__ :     Optional[int] 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)
									snake_case__ :     Tuple 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)
									snake_case__ :     Any 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)

									snake_case__ :     Dict 							=				{
									    'input_ids': multiple_choice_inputs_ids,
									    'mc_token_ids': mc_token_ids,
									    'attention_mask': multiple_choice_input_mask,
									    'token_type_ids': multiple_choice_token_type_ids,
									}
									snake_case__ :     Tuple 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(
									    result.logits.shape   ,	(self.batch_size, self.num_choices, self.seq_length, self.vocab_size)		)
									self.parent.assertEqual(result.mc_logits.shape   ,	(self.batch_size, self.num_choices)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     int 							=				self.num_labels
									snake_case__ :     int 							=				ids_tensor([self.batch_size]   ,	self.type_sequence_label_size		)
									snake_case__ :     str 							=				{
									    'input_ids': input_ids,
									    'attention_mask': input_mask,
									    'token_type_ids': token_type_ids,
									    'labels': sequence_labels,
									}
									snake_case__ :     str 							=				TFOpenAIGPTForSequenceClassification(__UpperCamelCase		)
									snake_case__ :     int 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(result.logits.shape   ,	(self.batch_size, self.num_labels)		)




					def       __a      (       self		)					->  List[str]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				self.prepare_config_and_inputs()

									(
									    (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  
									) :     Any 							=				config_and_inputs

									snake_case__ :     List[str] 							=				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict





@require_tf
class 			__snake_case  (      _lowerCamelCase   ,_lowerCamelCase   ,unittest.TestCase     ):
					__lowerCamelCase	 =       (
					    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
					    if is_tf_available()
					    else ()
					)
					__lowerCamelCase	 =       (
					    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
					)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
					__lowerCamelCase	 =       (
					    {
					        """feature-extraction""": TFOpenAIGPTModel,
					        """text-classification""": TFOpenAIGPTForSequenceClassification,
					        """text-generation""": TFOpenAIGPTLMHeadModel,
					        """zero-shot""": TFOpenAIGPTForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					__lowerCamelCase	 =       False
					__lowerCamelCase	 =       False




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase		)					->  int:



									'''simple docstring'''







									if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
													# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
													# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
													# tiny config could not be created.
													return True

									return False




					def       __a      (       self		)					->  Any:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				TFOpenAIGPTModelTester(self		)
									snake_case__ :     Any 							=				ConfigTester(self   ,	config_class=__UpperCamelCase   ,	n_embd=37		)




					def       __a      (       self		)					->  int:



									'''simple docstring'''







									self.config_tester.run_common_tests()




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     Optional[Any] 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_model(*__UpperCamelCase		)




					def       __a      (       self		)					->  int:



									'''simple docstring'''







									snake_case__ :     str 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_lm_head(*__UpperCamelCase		)




					def       __a      (       self		)					->  List[Any]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_double_head(*__UpperCamelCase		)




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     str 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__UpperCamelCase		)




					@slow
					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													snake_case__ :     Optional[Any] 							=				TFOpenAIGPTModel.from_pretrained(__UpperCamelCase		)
													self.assertIsNotNone(__UpperCamelCase		)





@require_tf
class 			__snake_case  (      unittest.TestCase     ):




					@slow
					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt'		)
									snake_case__ :     Any 							=				tf.convert_to_tensor([[481, 4735, 544]]   ,	dtype=tf.intaa		)  # the president is
									snake_case__ :     int 							=				[
									    481,
									    4735,
									    544,
									    246,
									    963,
									    870,
									    762,
									    239,
									    244,
									    40477,
									    244,
									    249,
									    719,
									    881,
									    487,
									    544,
									    240,
									    244,
									    603,
									    481,
									]  # the president is a very good man. " \n " i\'m sure he is, " said the

									snake_case__ :     int 							=				model.generate(__UpperCamelCase   ,	do_sample=__UpperCamelCase		)
									self.assertListEqual(output_ids[0].numpy().tolist()   ,	__UpperCamelCase		)


