

'''simple docstring'''

from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCAmelCase							=					logging.get_logger(__name__)

UpperCAmelCase							=					{
    '''google/long-t5-local-base''': '''https://huggingface.co/google/long-t5-local-base/blob/main/config.json''',
    '''google/long-t5-local-large''': '''https://huggingface.co/google/long-t5-local-large/blob/main/config.json''',
    '''google/long-t5-tglobal-base''': '''https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json''',
    '''google/long-t5-tglobal-large''': '''https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json''',
}


class 					lowerCAmelCase       ( A				):
       lowerCAmelCase_				=	"longt5"
       lowerCAmelCase_				=	["past_key_values"]
       lowerCAmelCase_				=	{"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}







       def __init__(							self     :	Optional[int]  , __lowercase     :	str=32128  , __lowercase     :	str=512  , __lowercase     :	List[str]=64  , __lowercase     :	Optional[Any]=2048  , __lowercase     :	List[str]=6  , __lowercase     :	Optional[int]=None  , __lowercase     :	Optional[int]=8  , __lowercase     :	Optional[int]=127  , __lowercase     :	Dict=16  , __lowercase     :	List[str]=32  , __lowercase     :	Optional[Any]=128  , __lowercase     :	Tuple=0.1  , __lowercase     :	Optional[Any]=1E-6  , __lowercase     :	Dict=1.0  , __lowercase     :	Union[str, Any]="relu"  , __lowercase     :	Optional[int]=True  , __lowercase     :	Optional[int]="local"  , __lowercase     :	Optional[Any]=True  , __lowercase     :	Tuple=0  , __lowercase     :	Tuple=1  , **__lowercase     :	Union[str, Any]  , ):







            """simple docstring"""







            __lowercase     						=vocab_size
            __lowercase     						=d_model
            __lowercase     						=d_kv
            __lowercase     						=d_ff
            __lowercase     						=num_layers
            # default = symmetry
            __lowercase     						=num_decoder_layers if num_decoder_layers is not None else self.num_layers
            __lowercase     						=num_heads
            __lowercase     						=local_radius
            __lowercase     						=global_block_size
            __lowercase     						=relative_attention_num_buckets
            __lowercase     						=relative_attention_max_distance
            __lowercase     						=dropout_rate
            __lowercase     						=layer_norm_epsilon
            __lowercase     						=initializer_factor
            __lowercase     						=feed_forward_proj
            __lowercase     						=encoder_attention_type
            __lowercase     						=use_cache

            __lowercase     						=self.feed_forward_proj.split('-' )
            __lowercase     						=act_info[-1]
            __lowercase     						=act_info[0] == 'gated'

            if len(__lowercase ) > 1 and act_info[0] != "gated" or len(__lowercase ) > 2:
                 raise ValueError(
                     f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
                     'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
                     '\'gated-gelu\' or \'relu\'' )

            # for backwards compatibility
            if feed_forward_proj == "gated-gelu":
                 __lowercase     						='gelu_new'

            super().__init__(
                pad_token_id=__lowercase  , eos_token_id=__lowercase  , is_encoder_decoder=__lowercase  , **__lowercase  , )






class 					lowerCAmelCase       ( A				):







       @property
       def      snake_case				(							self     :	Optional[int] ):







            """simple docstring"""







            __lowercase     						={
                'input_ids': {0: 'batch', 1: 'encoder_sequence'},
                'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
            }
            if self.use_past:
                 __lowercase     						='past_encoder_sequence + sequence'
                 __lowercase     						={0: 'batch'}
                 __lowercase     						={0: 'batch', 1: 'past_decoder_sequence + sequence'}
            else:
                 __lowercase     						={0: 'batch', 1: 'decoder_sequence'}
                 __lowercase     						={0: 'batch', 1: 'decoder_sequence'}

            if self.use_past:
                 self.fill_with_past_key_values_(__lowercase  , direction='inputs' )

            return common_inputs







       @property
       def      snake_case				(							self     :	Any ):







            """simple docstring"""







            return 13





'''simple docstring'''

import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


UpperCAmelCase							=					logging.get_logger(__name__)

UpperCAmelCase							=					[
    '''yjernite/retribert-base-uncased''',
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]


class 					lowerCAmelCase       ( A				):
       lowerCAmelCase_				=	RetriBertConfig
       lowerCAmelCase_				=	None
       lowerCAmelCase_				=	"retribert"







       def      snake_case				(							self     :	Tuple  , __lowercase     :	List[str] ):







            """simple docstring"""







            if isinstance(__lowercase  , nn.Linear ):
                 module.weight.data.normal_(mean=0.0  , std=self.config.initializer_range )
                 if module.bias is not None:
                      module.bias.data.zero_()
            elif isinstance(__lowercase  , nn.Embedding ):
                 module.weight.data.normal_(mean=0.0  , std=self.config.initializer_range )
                 if module.padding_idx is not None:
                      module.weight.data[module.padding_idx].zero_()
            elif isinstance(__lowercase  , nn.LayerNorm ):
                 module.bias.data.zero_()
                 module.weight.data.fill_(1.0 )






UpperCAmelCase							=					r'''

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''


@add_start_docstrings(
    "Bert Based model to embed queries or document for document retrieval."				,     A				,     )
class 					lowerCAmelCase       ( A				):







       def __init__(							self     :	int  , __lowercase     :	RetriBertConfig ):







            """simple docstring"""







            super().__init__(__lowercase )
            __lowercase     						=config.projection_dim

            __lowercase     						=BertModel(__lowercase )
            __lowercase     						=None if config.share_encoders else BertModel(__lowercase )
            __lowercase     						=nn.Dropout(config.hidden_dropout_prob )
            __lowercase     						=nn.Linear(config.hidden_size  , config.projection_dim  , bias=__lowercase )
            __lowercase     						=nn.Linear(config.hidden_size  , config.projection_dim  , bias=__lowercase )

            __lowercase     						=nn.CrossEntropyLoss(reduction='mean' )

            # Initialize weights and apply final processing
            self.post_init()







       def      snake_case				(							self     :	Dict  , __lowercase     :	List[Any]  , __lowercase     :	Any  , __lowercase     :	Optional[int]  , __lowercase     :	str=-1  , ):







            """simple docstring"""







            if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
                 return sent_encoder(__lowercase  , attention_mask=__lowercase )[1]
            else:
                 # prepare implicit variables
                 __lowercase     						=input_ids.device
                 __lowercase     						=input_ids.size()
                 __lowercase     						=torch.zeros(__lowercase  , dtype=torch.long  , device=__lowercase )
                 __lowercase     						=[None] * sent_encoder.config.num_hidden_layers
                 __lowercase     						=sent_encoder.get_extended_attention_mask(
                     __lowercase  , __lowercase )

                 # define function for checkpointing
                 def partial_encode(*__lowercase     :	int ):
                      __lowercase     						=sent_encoder.encoder(
                          inputs[0]  , attention_mask=inputs[1]  , head_mask=__lowercase  , )
                      __lowercase     						=encoder_outputs[0]
                      __lowercase     						=sent_encoder.pooler(__lowercase )
                      return pooled_output

                 # run embedding layer on everything at once
                 __lowercase     						=sent_encoder.embeddings(
                     input_ids=__lowercase  , position_ids=__lowercase  , token_type_ids=__lowercase  , inputs_embeds=__lowercase )
                 # run encoding and pooling on one mini-batch at a time
                 __lowercase     						=[]
                 for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size ) ):
                      __lowercase     						=embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                      __lowercase     						=extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                      __lowercase     						=checkpoint.checkpoint(__lowercase  , __lowercase  , __lowercase )
                      pooled_output_list.append(__lowercase )
                 return torch.cat(__lowercase  , dim=0 )







       def      snake_case				(							self     :	List[Any]  , __lowercase     :	Any  , __lowercase     :	Any=None  , __lowercase     :	Optional[int]=-1  , ):







            """simple docstring"""







            __lowercase     						=self.embed_sentences_checkpointed(
                __lowercase  , __lowercase  , self.bert_query  , __lowercase  , )
            return self.project_query(__lowercase )







       def      snake_case				(							self     :	Dict  , __lowercase     :	Tuple  , __lowercase     :	Optional[int]=None  , __lowercase     :	Any=-1  , ):







            """simple docstring"""







            __lowercase     						=self.embed_sentences_checkpointed(
                __lowercase  , __lowercase  , self.bert_query if self.bert_doc is None else self.bert_doc  , __lowercase  , )
            return self.project_doc(__lowercase )







       def      snake_case				(							self     :	int  , __lowercase     :	torch.LongTensor  , __lowercase     :	Optional[torch.FloatTensor]  , __lowercase     :	torch.LongTensor  , __lowercase     :	Optional[torch.FloatTensor]  , __lowercase     :	int = -1  , ):







            """simple docstring"""







            __lowercase     						=input_ids_query.device
            __lowercase     						=self.embed_questions(__lowercase  , __lowercase  , __lowercase )
            __lowercase     						=self.embed_answers(__lowercase  , __lowercase  , __lowercase )
            __lowercase     						=torch.mm(__lowercase  , a_reps.t() )
            __lowercase     						=self.ce_loss(__lowercase  , torch.arange(compare_scores.shape[1] ).to(__lowercase ) )
            __lowercase     						=self.ce_loss(compare_scores.t()  , torch.arange(compare_scores.shape[0] ).to(__lowercase ) )
            __lowercase     						=(loss_qa + loss_aq) / 2
            return loss


