


import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


UpperCAmelCase__            =							logging.get_logger(__name__)







class 				lowerCamelCase__       (	nn.Module):





     def __init__(self	)					->       List[Any]:
      super().__init__()
      if version.parse(torch.__version__	) < version.parse('''1.12.0'''	):
       raise ImportError(
           f"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use "
           '''PytorchGELUTanh. Please upgrade torch.'''	)





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return nn.functional.gelu(UpperCAmelCase		,					approximate='''tanh'''	)







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi	) * (input + 0.04_4715 * torch.pow(UpperCAmelCase		,					3.0	))	))







class 				lowerCamelCase__       (	nn.Module):





     def __init__(self		,					UpperCAmelCase = False	)					->       Union[str, Any]:
      super().__init__()
      if use_gelu_python:
       _lowercase											=self._gelu_python
      else:
       _lowercase											=nn.functional.gelu





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0	)	))





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return self.act(UpperCAmelCase	)







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)	))







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return input * torch.sigmoid(1.702 * input	)







class 				lowerCamelCase__       (	nn.Module):





     def __init__(self		,					UpperCAmelCase		,					UpperCAmelCase	)					->       Dict:
      if min > max:
       raise ValueError(f"min should be < max (got min: {min}, max: {max})"	)

      super().__init__()
      _lowercase											=min
      _lowercase											=max





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return torch.clip(gelu(UpperCAmelCase	)		,					self.min		,					self.max	)







class 				lowerCamelCase__       (	nn.Module):





     def __init__(self	)					->       Union[str, Any]:
      super().__init__()
      _lowercase											=math.sqrt(2 / math.pi	)





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(UpperCAmelCase		,					3	))	))







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return nn.functional.silu(UpperCAmelCase	)







class 				lowerCamelCase__       (	nn.Module):





     def __init__(self	)					->       Union[str, Any]:
      super().__init__()
      if version.parse(torch.__version__	) < version.parse('''1.9.0'''	):
       _lowercase											=self._mish_python
      else:
       _lowercase											=nn.functional.mish





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return input * torch.tanh(nn.functional.softplus(UpperCAmelCase	)	)





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return self.act(UpperCAmelCase	)







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Tensor:
      return input







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase=0.70_7107		,					UpperCAmelCase=0.28_2095	)					->       str:
      _lowercase											=(input - mu).div(sigma * math.sqrt(2.0	)	)
      return 0.5 * (1.0 + torch.erf(UpperCAmelCase	))







class 				lowerCamelCase__       (	nn.Module):





     def 					__A (self		,					UpperCAmelCase	)					->       Optional[int]:
      _lowercase											=nn.functional.relu(UpperCAmelCase	)
      _lowercase											=torch.square(UpperCAmelCase	)
      return squared







class 				lowerCamelCase__       (	lowerCAmelCase):





     def __getitem__(self		,					UpperCAmelCase	)					->       Union[str, Any]:
      _lowercase											=super().__getitem__(UpperCAmelCase	)
      _lowercase				,   _lowercase											=content if isinstance(UpperCAmelCase		,					UpperCAmelCase	) else (content, {})
      return cls(**UpperCAmelCase	)



UpperCAmelCase__            =							{
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
UpperCAmelCase__            =							ClassInstantier(ACTaCLS)
def     UpperCAmelCase_ (							__snake_case						)					->      str:







 """simple docstring"""


 if activation_string in ACTaFN:
  return ACTaFN[activation_string]
 else:
  raise KeyError(F"function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}"						)


# For backwards compatibility with: from activations import gelu_python
UpperCAmelCase__            =							get_activation('''gelu_python''')
UpperCAmelCase__            =							get_activation('''gelu_new''')
UpperCAmelCase__            =							get_activation('''gelu''')
UpperCAmelCase__            =							get_activation('''gelu_fast''')
UpperCAmelCase__            =							get_activation('''quick_gelu''')
UpperCAmelCase__            =							get_activation('''silu''')
UpperCAmelCase__            =							get_activation('''mish''')



UpperCAmelCase__            =							get_activation('''linear''')





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase__            =							{'''configuration_visual_bert''': ['''VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''VisualBertConfig''']}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__            =							[
      '''VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''VisualBertForMultipleChoice''',
      '''VisualBertForPreTraining''',
      '''VisualBertForQuestionAnswering''',
      '''VisualBertForRegionToPhraseAlignment''',
      '''VisualBertForVisualReasoning''',
      '''VisualBertLayer''',
      '''VisualBertModel''',
      '''VisualBertPreTrainedModel''',
  ]


if TYPE_CHECKING:
  from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_visual_bert import (
        VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        VisualBertForMultipleChoice,
        VisualBertForPreTraining,
        VisualBertForQuestionAnswering,
        VisualBertForRegionToPhraseAlignment,
        VisualBertForVisualReasoning,
        VisualBertLayer,
        VisualBertModel,
        VisualBertPreTrainedModel,
    )


else:
  import sys

  UpperCAmelCase__            =							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

