





import torch

from accelerate import Accelerator, DistributedType






class  __A   :

			'''simple docstring'''







			def __enter__(   self							):



				'''simple docstring'''







				if self.enabled:
					lowerCamelCase__        =		self.model.no_sync()
					self.model_sync_obj.__enter__()

				return self





			def __exit__(   self	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase							):



				'''simple docstring'''







				if self.enabled:
					# Average all models on exit
					self._sync_and_avg_model_params()
					self.model_sync_obj.__exit__(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase							)





			def __init__(   self	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase = True							):



				'''simple docstring'''







				if accelerator.distributed_type not in [
				    DistributedType.NO,
				    DistributedType.MULTI_CPU,
				    DistributedType.MULTI_GPU,
				]:
					raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''							)
				lowerCamelCase__        =		enabled and accelerator.distributed_type != DistributedType.NO
				lowerCamelCase__        =		0
				if self.enabled:
					lowerCamelCase__        =		accelerator
					lowerCamelCase__        =		model
					lowerCamelCase__        =		local_sgd_steps





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				self.num_steps += 1
				if not self.enabled:
					return

				if self.num_steps % self.local_sgd_steps == 0:
					self._sync_and_avg_model_params()





			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				self.accelerator.wait_for_everyone()
				with self.accelerator.autocast():
					for param in self.model.parameters():
						lowerCamelCase__        =		self.accelerator.reduce(param.data	,  reduction='''mean'''							)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a									=				logging.get_logger(__name__)

_a									=				{
    "google/pegasus-x-base": "https://huggingface.co/google/pegasus-x-base/resolve/main/config.json",
    "google/pegasus-x-large": "https://huggingface.co/google/pegasus-x-large/resolve/main/config.json",
    # See all PEGASUS-X models at https://huggingface.co/models?filter=pegasus-x
}






class  __A   (   lowerCAmelCase			):

			'''simple docstring'''







			lowerCAmelCase_												=						"""pegasus_x"""
			lowerCAmelCase_												=						["""past_key_values"""]
			lowerCAmelCase_												=						{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}





			def __init__(   self	,  __lowerCAmelCase=9_6_1_0_3	,  __lowerCAmelCase=1_6_3_8_4	,  __lowerCAmelCase=1_6	,  __lowerCAmelCase=4_0_9_6	,  __lowerCAmelCase=1_6	,  __lowerCAmelCase=1_6	,  __lowerCAmelCase=4_0_9_6	,  __lowerCAmelCase=1_6	,  __lowerCAmelCase=0.0	,  __lowerCAmelCase=0.0	,  __lowerCAmelCase=True	,  __lowerCAmelCase=True	,  __lowerCAmelCase="gelu"	,  __lowerCAmelCase=1_0_2_4	,  __lowerCAmelCase=0.1	,  __lowerCAmelCase=0.0	,  __lowerCAmelCase=0.0	,  __lowerCAmelCase=0.02	,  __lowerCAmelCase=0	,  __lowerCAmelCase=True	,  __lowerCAmelCase=0	,  __lowerCAmelCase=1	,  __lowerCAmelCase=1	,  __lowerCAmelCase=3_2	,  __lowerCAmelCase=5_1_2	,  __lowerCAmelCase=True	,  **__lowerCAmelCase	,  ):



				'''simple docstring'''







				lowerCamelCase__        =		vocab_size
				lowerCamelCase__        =		max_position_embeddings
				lowerCamelCase__        =		d_model
				lowerCamelCase__        =		encoder_ffn_dim
				lowerCamelCase__        =		encoder_layers
				lowerCamelCase__        =		encoder_attention_heads
				lowerCamelCase__        =		decoder_ffn_dim
				lowerCamelCase__        =		decoder_layers
				lowerCamelCase__        =		decoder_attention_heads
				lowerCamelCase__        =		dropout
				lowerCamelCase__        =		attention_dropout
				lowerCamelCase__        =		activation_dropout
				lowerCamelCase__        =		activation_function
				lowerCamelCase__        =		init_std
				lowerCamelCase__        =		encoder_layerdrop
				lowerCamelCase__        =		decoder_layerdrop
				lowerCamelCase__        =		use_cache
				lowerCamelCase__        =		encoder_layers
				lowerCamelCase__        =		scale_embedding  # scale factor will be sqrt(d_model) if True

				lowerCamelCase__        =		num_global_tokens
				lowerCamelCase__        =		block_size
				lowerCamelCase__        =		stagger_local_blocks

				super().__init__(
				    pad_token_id=__lowerCAmelCase	,  eos_token_id=__lowerCAmelCase	,  is_encoder_decoder=__lowerCAmelCase	,  decoder_start_token_id=__lowerCAmelCase	,  forced_eos_token_id=__lowerCAmelCase	,  **__lowerCAmelCase	,  )





			@property
			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return self.encoder_attention_heads





			@property
			def 		__lowerCamelCase    (   self							):



				'''simple docstring'''







				return self.d_model



