


def      _lowerCAmelCase  (							lowerCAmelCase_							:list							)->list:
   '''simple docstring'''






   snake_case_         =   len(lowerCAmelCase_							)

   # Set the gap value to be decreased by a factor of 1.3
   # after each iteration
   snake_case_         =   1.3

   # Continue sorting until the gap is 1
   while gap > 1:
      # Decrease the gap value
      snake_case_         =   int(gap / shrink							)

      # Sort the elements using insertion sort
      for i in range(lowerCAmelCase_						,  len(lowerCAmelCase_							)							):
         snake_case_         =   collection[i]
         snake_case_         =   i
         while j >= gap and collection[j - gap] > temp:
            snake_case_         =   collection[j - gap]
            j -= gap
         snake_case_         =   temp

   return collection


if __name__ == "__main__":
   import doctest

   doctest.testmod()








import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


SCREAMING_SNAKE_CASE		:Tuple											=       logging.get_logger(__name__)


class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def __init__(       self    :    Any					)				->	List[str]:




     """simple docstring"""
     super().__init__()
     if version.parse(torch.__version__					) < version.parse("1.12.0"					):
        raise ImportError(
            F'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
            "PytorchGELUTanh. Please upgrade torch."					)



  def    lowerCAmelCase__					(       self    :    Optional[int]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return nn.functional.gelu(_lowerCAmelCase      ,			approximate="tanh"					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    int      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi					) * (input + 0.044_715 * torch.pow(_lowerCAmelCase      ,			3.0					))					))






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def __init__(       self    :    int      ,			_lowerCAmelCase    :    bool = False					)				->	int:




     """simple docstring"""
     super().__init__()
     if use_gelu_python:
        snake_case_         =   self._gelu_python
     else:
        snake_case_         =   nn.functional.gelu



  def    lowerCAmelCase__					(       self    :    Optional[Any]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0					)					))



  def    lowerCAmelCase__					(       self    :    Dict      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return self.act(_lowerCAmelCase					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    Optional[int]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)					))






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    int      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return input * torch.sigmoid(1.702 * input					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def __init__(       self    :    List[str]      ,			_lowerCAmelCase    :    float      ,			_lowerCAmelCase    :    float					)				->	Any:




     """simple docstring"""
     if min > max:
        raise ValueError(F'''min should be < max (got min: {min}, max: {max})'''					)

     super().__init__()
     snake_case_         =   min
     snake_case_         =   max



  def    lowerCAmelCase__					(       self    :    Union[str, Any]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return torch.clip(gelu(_lowerCAmelCase					)      ,			self.min      ,			self.max					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def __init__(       self    :    Any					)				->	str:




     """simple docstring"""
     super().__init__()
     snake_case_         =   math.sqrt(2 / math.pi					)



  def    lowerCAmelCase__					(       self    :    Optional[Any]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(_lowerCAmelCase      ,			3					))					))






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    str      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return nn.functional.silu(_lowerCAmelCase					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def __init__(       self    :    Optional[int]					)				->	List[Any]:




     """simple docstring"""
     super().__init__()
     if version.parse(torch.__version__					) < version.parse("1.9.0"					):
        snake_case_         =   self._mish_python
     else:
        snake_case_         =   nn.functional.mish



  def    lowerCAmelCase__					(       self    :    Tuple      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return input * torch.tanh(nn.functional.softplus(_lowerCAmelCase					)					)



  def    lowerCAmelCase__					(       self    :    Optional[Any]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return self.act(_lowerCAmelCase					)






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    List[str]      ,			_lowerCAmelCase    :    Tensor					)				->	Tensor:




     """simple docstring"""
     return input






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    List[Any]      ,			_lowerCAmelCase    :    List[Any]      ,			_lowerCAmelCase    :    Union[str, Any]=0.707_107      ,			_lowerCAmelCase    :    int=0.282_095					)				->	List[str]:




     """simple docstring"""
     snake_case_         =   (input - mu).div(sigma * math.sqrt(2.0					)					)
     return 0.5 * (1.0 + torch.erf(_lowerCAmelCase					))






class       __lowerCAmelCase       (	nn.Module   ):






  """simple docstring"""





  def    lowerCAmelCase__					(       self    :    Any      ,			_lowerCAmelCase    :    Union[str, Any]					)				->	Any:




     """simple docstring"""
     snake_case_         =   nn.functional.relu(_lowerCAmelCase					)
     snake_case_         =   torch.square(_lowerCAmelCase					)
     return squared






class       __lowerCAmelCase       (	a   ):






  """simple docstring"""





  def __getitem__(       self    :    Optional[Any]      ,			_lowerCAmelCase    :    str					)				->	Optional[int]:




     """simple docstring"""
     snake_case_         =   super().__getitem__(_lowerCAmelCase					)
     snake_case_      ,      snake_case_         =   content if isinstance(_lowerCAmelCase      ,			_lowerCAmelCase					) else (content, {})
     return cls(**_lowerCAmelCase					)






SCREAMING_SNAKE_CASE		:Optional[int]											=       {
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
SCREAMING_SNAKE_CASE		:Optional[int]											=       ClassInstantier(ACTaCLS)



def      _lowerCAmelCase  (							lowerCAmelCase_							:Optional[Any]							)->Union[str, Any]:
   '''simple docstring'''






   if activation_string in ACTaFN:
      return ACTaFN[activation_string]
   else:
      raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()							)}'''							)


# For backwards compatibility with: from activations import gelu_python
SCREAMING_SNAKE_CASE		:Tuple											=       get_activation('''gelu_python''')
SCREAMING_SNAKE_CASE		:int											=       get_activation('''gelu_new''')
SCREAMING_SNAKE_CASE		:List[Any]											=       get_activation('''gelu''')
SCREAMING_SNAKE_CASE		:int											=       get_activation('''gelu_fast''')
SCREAMING_SNAKE_CASE		:Tuple											=       get_activation('''quick_gelu''')
SCREAMING_SNAKE_CASE		:Tuple											=       get_activation('''silu''')
SCREAMING_SNAKE_CASE		:Any											=       get_activation('''mish''')



SCREAMING_SNAKE_CASE		:Optional[Any]											=       get_activation('''linear''')




