
'''simple docstring'''



import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.semantic_stable_diffusion import SemanticStableDiffusionPipeline as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class 				_a			(unittest.TestCase):






       def     UpperCamelCase			(							self )      ->     Dict:
          # clean up the VRAM after each test
          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()






       @property
       def     UpperCamelCase			(							self )      ->     List[str]:
          _SCREAMING_SNAKE_CASE       	=			1
          _SCREAMING_SNAKE_CASE       	=			3
          _SCREAMING_SNAKE_CASE       	=			(32, 32)

          _SCREAMING_SNAKE_CASE       	=			floats_tensor((batch_size, num_channels) + sizes    ,							rng=random.Random(0 ) ).to(A__ )
          return image






       @property
       def     UpperCamelCase			(							self )      ->     Optional[int]:
          torch.manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			UNetaDConditionModel(
              block_out_channels=(32, 64)    ,							layers_per_block=2    ,							sample_size=32    ,							in_channels=4    ,							out_channels=4    ,							down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")    ,							up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")    ,							cross_attention_dim=32    ,							)
          return model






       @property
       def     UpperCamelCase			(							self )      ->     str:
          torch.manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			AutoencoderKL(
              block_out_channels=[32, 64]    ,							in_channels=3    ,							out_channels=3    ,							down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]    ,							up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]    ,							latent_channels=4    ,							)
          return model






       @property
       def     UpperCamelCase			(							self )      ->     Dict:
          torch.manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			CLIPTextConfig(
              bos_token_id=0    ,							eos_token_id=2    ,							hidden_size=32    ,							intermediate_size=37    ,							layer_norm_eps=1E-05    ,							num_attention_heads=4    ,							num_hidden_layers=5    ,							pad_token_id=1    ,							vocab_size=10_00    ,							)
          return CLIPTextModel(A__ )






       @property
       def     UpperCamelCase			(							self )      ->     Optional[Any]:
          def extract(*A__    ,							**A__ ):

             class 				_a			:






                    def __init__(							self )      ->     Optional[Any]:
                       _SCREAMING_SNAKE_CASE       	=			torch.ones([0] )






                    def     UpperCamelCase			(							self    ,							A__ )      ->     int:
                       self.pixel_values.to(A__ )
                       return self






             return Out()

          return extract






       def     UpperCamelCase			(							self )      ->     Union[str, Any]:
          _SCREAMING_SNAKE_CASE       	=			"""cpu"""  # ensure determinism for the device-dependent torch.Generator
          _SCREAMING_SNAKE_CASE       	=			self.dummy_cond_unet
          _SCREAMING_SNAKE_CASE       	=			DDIMScheduler(
              beta_start=0.0_0085    ,							beta_end=0.012    ,							beta_schedule="""scaled_linear"""    ,							clip_sample=A__    ,							set_alpha_to_one=A__    ,							)

          _SCREAMING_SNAKE_CASE       	=			self.dummy_vae
          _SCREAMING_SNAKE_CASE       	=			self.dummy_text_encoder
          _SCREAMING_SNAKE_CASE       	=			CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip""" )

          # make sure here that pndm scheduler skips prk
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline(
              unet=A__    ,							scheduler=A__    ,							vae=A__    ,							text_encoder=A__    ,							tokenizer=A__    ,							safety_checker=A__    ,							feature_extractor=self.dummy_extractor    ,							)
          _SCREAMING_SNAKE_CASE       	=			sd_pipe.to(A__ )
          sd_pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""A painting of a squirrel eating a burger"""

          _SCREAMING_SNAKE_CASE       	=			torch.Generator(device=A__ ).manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			sd_pipe([prompt]    ,							generator=A__    ,							guidance_scale=6.0    ,							num_inference_steps=2    ,							output_type="""np""" )
          _SCREAMING_SNAKE_CASE       	=			output.images

          _SCREAMING_SNAKE_CASE       	=			torch.Generator(device=A__ ).manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			sd_pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=6.0    ,							num_inference_steps=2    ,							output_type="""np"""    ,							return_dict=A__    ,							)[0]

          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			image_from_tuple[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)
          _SCREAMING_SNAKE_CASE       	=			np.array([0.5753, 0.6114, 0.5001, 0.5034, 0.5470, 0.4729, 0.4971, 0.4867, 0.4867] )

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2
          assert np.abs(image_from_tuple_slice.flatten() - expected_slice ).max() < 1E-2






       def     UpperCamelCase			(							self )      ->     str:
          _SCREAMING_SNAKE_CASE       	=			"""cpu"""  # ensure determinism for the device-dependent torch.Generator
          _SCREAMING_SNAKE_CASE       	=			self.dummy_cond_unet
          _SCREAMING_SNAKE_CASE       	=			PNDMScheduler(skip_prk_steps=A__ )
          _SCREAMING_SNAKE_CASE       	=			self.dummy_vae
          _SCREAMING_SNAKE_CASE       	=			self.dummy_text_encoder
          _SCREAMING_SNAKE_CASE       	=			CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip""" )

          # make sure here that pndm scheduler skips prk
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline(
              unet=A__    ,							scheduler=A__    ,							vae=A__    ,							text_encoder=A__    ,							tokenizer=A__    ,							safety_checker=A__    ,							feature_extractor=self.dummy_extractor    ,							)
          _SCREAMING_SNAKE_CASE       	=			sd_pipe.to(A__ )
          sd_pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""A painting of a squirrel eating a burger"""
          _SCREAMING_SNAKE_CASE       	=			torch.Generator(device=A__ ).manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			sd_pipe([prompt]    ,							generator=A__    ,							guidance_scale=6.0    ,							num_inference_steps=2    ,							output_type="""np""" )

          _SCREAMING_SNAKE_CASE       	=			output.images

          _SCREAMING_SNAKE_CASE       	=			torch.Generator(device=A__ ).manual_seed(0 )
          _SCREAMING_SNAKE_CASE       	=			sd_pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=6.0    ,							num_inference_steps=2    ,							output_type="""np"""    ,							return_dict=A__    ,							)[0]

          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			image_from_tuple[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)
          _SCREAMING_SNAKE_CASE       	=			np.array([0.5122, 0.5712, 0.4825, 0.5053, 0.5646, 0.4769, 0.5179, 0.4894, 0.4994] )

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2
          assert np.abs(image_from_tuple_slice.flatten() - expected_slice ).max() < 1E-2






       def     UpperCamelCase			(							self )      ->     Optional[int]:
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained(
              """hf-internal-testing/tiny-stable-diffusion-lms-pipe"""    ,							safety_checker=A__ )
          assert isinstance(A__    ,							A__ )
          assert isinstance(pipe.scheduler    ,							A__ )
          assert pipe.safety_checker is None

          _SCREAMING_SNAKE_CASE       	=			pipe("""example prompt"""    ,							num_inference_steps=2 ).images[0]
          assert image is not None

          # check that there's no error when saving a pipeline with one of the models being None
          with tempfile.TemporaryDirectory() as tmpdirname:
             pipe.save_pretrained(A__ )
             _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained(A__ )

          # sanity check that the pipeline still works
          assert pipe.safety_checker is None
          _SCREAMING_SNAKE_CASE       	=			pipe("""example prompt"""    ,							num_inference_steps=2 ).images[0]
          assert image is not None






       @unittest.skipIf(torch_device != """cuda"""    ,							"""This test requires a GPU""" )
       def     UpperCamelCase			(							self )      ->     Union[str, Any]:
          _SCREAMING_SNAKE_CASE       	=			self.dummy_cond_unet
          _SCREAMING_SNAKE_CASE       	=			PNDMScheduler(skip_prk_steps=A__ )
          _SCREAMING_SNAKE_CASE       	=			self.dummy_vae
          _SCREAMING_SNAKE_CASE       	=			self.dummy_text_encoder
          _SCREAMING_SNAKE_CASE       	=			CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip""" )

          # put models in fp16
          _SCREAMING_SNAKE_CASE       	=			unet.half()
          _SCREAMING_SNAKE_CASE       	=			vae.half()
          _SCREAMING_SNAKE_CASE       	=			bert.half()

          # make sure here that pndm scheduler skips prk
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline(
              unet=A__    ,							scheduler=A__    ,							vae=A__    ,							text_encoder=A__    ,							tokenizer=A__    ,							safety_checker=A__    ,							feature_extractor=self.dummy_extractor    ,							)
          _SCREAMING_SNAKE_CASE       	=			sd_pipe.to(A__ )
          sd_pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""A painting of a squirrel eating a burger"""
          _SCREAMING_SNAKE_CASE       	=			sd_pipe([prompt]    ,							num_inference_steps=2    ,							output_type="""np""" ).images

          assert image.shape == (1, 64, 64, 3)






@nightly
@require_torch_gpu
class 				_a			(unittest.TestCase):






       def     UpperCamelCase			(							self )      ->     Dict:
          # clean up the VRAM after each test
          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()






       def     UpperCamelCase			(							self )      ->     Union[str, Any]:
          _SCREAMING_SNAKE_CASE       	=			"""cuda"""
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5""" )
          _SCREAMING_SNAKE_CASE       	=			pipe.to(A__ )
          pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""a photo of a cat"""
          _SCREAMING_SNAKE_CASE       	=			{
              """editing_prompt""": ["""sunglasses"""],
              """reverse_editing_direction""": [False],
              """edit_warmup_steps""": 10,
              """edit_guidance_scale""": 6,
              """edit_threshold""": 0.95,
              """edit_momentum_scale""": 0.5,
              """edit_mom_beta""": 0.6,
          }

          _SCREAMING_SNAKE_CASE       	=			3
          _SCREAMING_SNAKE_CASE       	=			7

          # no sega enabled
          _SCREAMING_SNAKE_CASE       	=			torch.Generator(A__ )
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.3467_3113,
              0.3849_2733,
              0.3759_7352,
              0.3408_6335,
              0.3565_0748,
              0.3557_9205,
              0.338_4763,
              0.3434_0236,
              0.357_3271,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2

          # with sega enabled
          # generator = torch.manual_seed(seed)
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							**A__    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.4188_7826,
              0.3772_8766,
              0.3013_8272,
              0.4141_6335,
              0.4166_4985,
              0.3628_3392,
              0.3619_1246,
              0.4336_4465,
              0.4300_1732,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2






       def     UpperCamelCase			(							self )      ->     Optional[int]:
          _SCREAMING_SNAKE_CASE       	=			"""cuda"""
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5""" )
          _SCREAMING_SNAKE_CASE       	=			pipe.to(A__ )
          pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""an image of a crowded boulevard, realistic, 4k"""
          _SCREAMING_SNAKE_CASE       	=			{
              """editing_prompt""": """crowd, crowded, people""",
              """reverse_editing_direction""": True,
              """edit_warmup_steps""": 10,
              """edit_guidance_scale""": 8.3,
              """edit_threshold""": 0.9,
              """edit_momentum_scale""": 0.5,
              """edit_mom_beta""": 0.6,
          }

          _SCREAMING_SNAKE_CASE       	=			9
          _SCREAMING_SNAKE_CASE       	=			7

          # no sega enabled
          _SCREAMING_SNAKE_CASE       	=			torch.Generator(A__ )
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.4349_7998,
              0.9181_4065,
              0.754_0739,
              0.5558_0205,
              0.846_7265,
              0.538_9691,
              0.6257_4506,
              0.5889_7763,
              0.5092_6757,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2

          # with sega enabled
          # generator = torch.manual_seed(seed)
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							**A__    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.308_9719,
              0.3050_0144,
              0.2901_6042,
              0.3063_0964,
              0.32_5687,
              0.2941_9225,
              0.290_8091,
              0.2872_3598,
              0.2769_6294,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2






       def     UpperCamelCase			(							self )      ->     Optional[int]:
          _SCREAMING_SNAKE_CASE       	=			"""cuda"""
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5""" )
          _SCREAMING_SNAKE_CASE       	=			pipe.to(A__ )
          pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""a castle next to a river"""
          _SCREAMING_SNAKE_CASE       	=			{
              """editing_prompt""": ["""boat on a river, boat""", """monet, impression, sunrise"""],
              """reverse_editing_direction""": False,
              """edit_warmup_steps""": [15, 18],
              """edit_guidance_scale""": 6,
              """edit_threshold""": [0.9, 0.8],
              """edit_momentum_scale""": 0.5,
              """edit_mom_beta""": 0.6,
          }

          _SCREAMING_SNAKE_CASE       	=			48
          _SCREAMING_SNAKE_CASE       	=			7

          # no sega enabled
          _SCREAMING_SNAKE_CASE       	=			torch.Generator(A__ )
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.7516_3555,
              0.7603_7145,
              0.6_1785,
              0.918_9673,
              0.862_7701,
              0.8518_9694,
              0.851_2813,
              0.8701_2076,
              0.831_2857,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2

          # with sega enabled
          # generator = torch.manual_seed(seed)
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							**A__    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.7355_3365,
              0.753_7271,
              0.7434_1905,
              0.6648_0356,
              0.647_2925,
              0.6303_9416,
              0.6481_2905,
              0.674_9717,
              0.651_7102,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2






       def     UpperCamelCase			(							self )      ->     Any:
          _SCREAMING_SNAKE_CASE       	=			"""cuda"""
          _SCREAMING_SNAKE_CASE       	=			StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""    ,							torch_dtype=torch.floataa )
          _SCREAMING_SNAKE_CASE       	=			pipe.to(A__ )
          pipe.set_progress_bar_config(disable=A__ )

          _SCREAMING_SNAKE_CASE       	=			"""a photo of a cat"""
          _SCREAMING_SNAKE_CASE       	=			{
              """editing_prompt""": ["""sunglasses"""],
              """reverse_editing_direction""": [False],
              """edit_warmup_steps""": 10,
              """edit_guidance_scale""": 6,
              """edit_threshold""": 0.95,
              """edit_momentum_scale""": 0.5,
              """edit_mom_beta""": 0.6,
          }

          _SCREAMING_SNAKE_CASE       	=			3
          _SCREAMING_SNAKE_CASE       	=			7

          # no sega enabled
          _SCREAMING_SNAKE_CASE       	=			torch.Generator(A__ )
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.3488_7695,
              0.387_6953,
              0.375,
              0.3442_3828,
              0.358_1543,
              0.3571_7773,
              0.338_3789,
              0.3457_0312,
              0.35_9375,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2

          # with sega enabled
          # generator = torch.manual_seed(seed)
          generator.manual_seed(A__ )
          _SCREAMING_SNAKE_CASE       	=			pipe(
              [prompt]    ,							generator=A__    ,							guidance_scale=A__    ,							num_inference_steps=50    ,							output_type="""np"""    ,							width=5_12    ,							height=5_12    ,							**A__    ,							)

          _SCREAMING_SNAKE_CASE       	=			output.images
          _SCREAMING_SNAKE_CASE       	=			image[0, -3:, -3:, -1]
          _SCREAMING_SNAKE_CASE       	=			[
              0.4228_5156,
              0.3691_4062,
              0.2907_7148,
              0.4204_1016,
              0.4191_8945,
              0.3549_8047,
              0.361_8164,
              0.442_3828,
              0.4311_5234,
          ]

          assert image.shape == (1, 5_12, 5_12, 3)

          assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2








'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__ :						int    							=					logging.get_logger(__name__)

UpperCamelCase__ :						Any    							=					{
    "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/config.json"
    # See all CPMAnt models at https://huggingface.co/models?filter=cpmant
}

class 				_a			(_lowerCamelCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE =					'cpmant'






       def __init__(							self    ,							A__ = 3_07_20    ,							A__ = 40_96    ,							A__ = 32    ,							A__ = 1_28    ,							A__ = 1_02_40    ,							A__ = 48    ,							A__ = 0.0    ,							A__ = 5_12    ,							A__ = 20_48    ,							A__ = 1E-6    ,							A__ = 1.0    ,							A__ = 32    ,							A__ = 32    ,							A__ = 32    ,							A__ = True    ,							**A__    ,							)      ->     List[Any]:
          super().__init__(**A__ )
          _SCREAMING_SNAKE_CASE       	=			prompt_types
          _SCREAMING_SNAKE_CASE       	=			prompt_length
          _SCREAMING_SNAKE_CASE       	=			segment_types
          _SCREAMING_SNAKE_CASE       	=			hidden_size
          _SCREAMING_SNAKE_CASE       	=			num_attention_heads
          _SCREAMING_SNAKE_CASE       	=			dim_head
          _SCREAMING_SNAKE_CASE       	=			dim_ff
          _SCREAMING_SNAKE_CASE       	=			num_hidden_layers
          _SCREAMING_SNAKE_CASE       	=			position_bias_num_buckets
          _SCREAMING_SNAKE_CASE       	=			position_bias_max_distance
          _SCREAMING_SNAKE_CASE       	=			dropout_p
          _SCREAMING_SNAKE_CASE       	=			eps
          _SCREAMING_SNAKE_CASE       	=			use_cache
          _SCREAMING_SNAKE_CASE       	=			vocab_size
          _SCREAMING_SNAKE_CASE       	=			init_std






