
import os
import tempfile
import unittest

import numpy as np
import torch

from diffusers.models import ModelMixin, UNetaDConditionModel
from diffusers.models.attention_processor import AttnProcessor, LoRAAttnProcessor
from diffusers.utils import (
    floats_tensor,
    logging,
    skip_mps,
    torch_device,
)
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()

A__  :  Tuple       = logging.get_logger(__name__)


def       UpperCamelCase(  __UpperCamelCase							:		Optional[int]					,__UpperCamelCase							:		bool = True			):
		lowerCAmelCase_      :					Tuple     =			{}
		for name in model.attn_processors.keys():
				lowerCAmelCase_      :					Dict     =			name.endswith('''attn2.processor'''			) and not (
				    name.startswith('''transformer_in'''			) or '''temp_attentions''' in name.split('''.'''			)
				)
				lowerCAmelCase_      :					Optional[Any]     =			model.config.cross_attention_dim if has_cross_attention else None
				if name.startswith('''mid_block'''			):
						lowerCAmelCase_      :					int     =			model.config.block_out_channels[-1]
				elif name.startswith('''up_blocks'''			):
						lowerCAmelCase_      :					List[str]     =			int(name[len('''up_blocks.'''			)]			)
						lowerCAmelCase_      :					Optional[Any]     =			list(reversed(model.config.block_out_channels			)			)[block_id]
				elif name.startswith('''down_blocks'''			):
						lowerCAmelCase_      :					str     =			int(name[len('''down_blocks.'''			)]			)
						lowerCAmelCase_      :					Optional[Any]     =			model.config.block_out_channels[block_id]
				elif name.startswith('''transformer_in'''			):
						# Note that the `8 * ...` comes from: https://github.com/huggingface/diffusers/blob/7139f0e874f10b2463caa8cbd585762a309d12d6/src/diffusers/models/unet_3d_condition.py#L148
						lowerCAmelCase_      :					int     =			8 * model.config.attention_head_dim

				lowerCAmelCase_      :					str     =			LoRAAttnProcessor(hidden_size=__UpperCamelCase					,cross_attention_dim=__UpperCamelCase			)
				lowerCAmelCase_      :					List[str]     =			lora_attn_procs[name].to(model.device			)

				if mock_weights:
						# add 1 to weights to mock trained weights
						with torch.no_grad():
								lora_attn_procs[name].to_q_lora.up.weight += 1
								lora_attn_procs[name].to_k_lora.up.weight += 1
								lora_attn_procs[name].to_v_lora.up.weight += 1
								lora_attn_procs[name].to_out_lora.up.weight += 1

		return lora_attn_procs





@skip_mps
class     __snake_case						(					UpperCamelCase_						,UpperCamelCase_						,unittest.TestCase						):
							_a      				=			UNetaDConditionModel
							_a      				=			'''sample'''




							@property
							def        UpperCAmelCase__      (	self					:	Union[str, Any]):
									lowerCAmelCase_      :					Dict     =			4
									lowerCAmelCase_      :					Dict     =			4
									lowerCAmelCase_      :					Union[str, Any]     =			4
									lowerCAmelCase_      :					Union[str, Any]     =			(3_2, 3_2)

									lowerCAmelCase_      :					int     =			floats_tensor((batch_size, num_channels, num_frames) + sizes).to(A_)
									lowerCAmelCase_      :					int     =			torch.tensor([1_0]).to(A_)
									lowerCAmelCase_      :					List[str]     =			floats_tensor((batch_size, 4, 3_2)).to(A_)

									return {"sample": noise, "timestep": time_step, "encoder_hidden_states": encoder_hidden_states}




							@property
							def        UpperCAmelCase__      (	self					:	Dict):
									return (4, 4, 3_2, 3_2)




							@property
							def        UpperCAmelCase__      (	self					:	List[str]):
									return (4, 4, 3_2, 3_2)




							def        UpperCAmelCase__      (	self					:	Optional[Any]):
									lowerCAmelCase_      :					List[str]     =			{
									    '''block_out_channels''': (3_2, 6_4),
									    '''down_block_types''': (
									        '''CrossAttnDownBlock3D''',
									        '''DownBlock3D''',
									    ),
									    '''up_block_types''': ('''UpBlock3D''', '''CrossAttnUpBlock3D'''),
									    '''cross_attention_dim''': 3_2,
									    '''attention_head_dim''': 8,
									    '''out_channels''': 4,
									    '''in_channels''': 4,
									    '''layers_per_block''': 1,
									    '''sample_size''': 3_2,
									}
									lowerCAmelCase_      :					Optional[Any]     =			self.dummy_input
									return init_dict, inputs_dict




							@unittest.skipIf(
							    torch_device != '''cuda''' or not is_xformers_available()    ,    reason='''XFormers attention is only available with CUDA and `xformers` installed'''    ,    )
							def        UpperCAmelCase__      (	self					:	str):
									lowerCAmelCase_      :					Union[str, Any]     =			self.prepare_init_args_and_inputs_for_common()
									lowerCAmelCase_      :					Tuple     =			self.model_class(**A_)

									model.enable_xformers_memory_efficient_attention()

									assert (
									    model.mid_block.attentions[0].transformer_blocks[0].attna.processor.__class__.__name__
									    == "XFormersAttnProcessor"
									), "xformers is not enabled"




							def        UpperCAmelCase__      (	self					:	Union[str, Any]):
									lowerCAmelCase_      :					Optional[int]     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					List[Any]     =			3_2

									lowerCAmelCase_      :					int     =			self.model_class(**A_)
									model.to(A_)
									model.eval()

									with torch.no_grad():
											lowerCAmelCase_      :					List[str]     =			model(**A_)

											if isinstance(A_    ,    A_):
													lowerCAmelCase_      :					Optional[Any]     =			output.sample

									self.assertIsNotNone(A_)
									lowerCAmelCase_      :					int     =			inputs_dict['''sample'''].shape
									self.assertEqual(output.shape    ,    A_    ,    '''Input and output shapes do not match''')




							def        UpperCAmelCase__      (	self					:	int):
									lowerCAmelCase_      :					Union[str, Any]     =			self.prepare_init_args_and_inputs_for_common()
									lowerCAmelCase_      :					Union[str, Any]     =			self.model_class(**A_)
									model.to(A_)
									model.eval()

									with torch.no_grad():
											# Warmup pass when using mps (see #372)
											if torch_device == "mps" and isinstance(A_    ,    A_):
													model(**self.dummy_input)

											lowerCAmelCase_      :					List[Any]     =			model(**A_)
											if isinstance(A_    ,    A_):
													lowerCAmelCase_      :					Dict     =			first.sample

											lowerCAmelCase_      :					Tuple     =			model(**A_)
											if isinstance(A_    ,    A_):
													lowerCAmelCase_      :					List[str]     =			second.sample

									lowerCAmelCase_      :					Optional[Any]     =			first.cpu().numpy()
									lowerCAmelCase_      :					int     =			second.cpu().numpy()
									lowerCAmelCase_      :					Any     =			out_a[~np.isnan(A_)]
									lowerCAmelCase_      :					Dict     =			out_a[~np.isnan(A_)]
									lowerCAmelCase_      :					Any     =			np.amax(np.abs(out_a - out_a))
									self.assertLessEqual(A_    ,    1e-5)




							def        UpperCAmelCase__      (	self					:	Union[str, Any]):
									lowerCAmelCase_      :					int     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					str     =			8

									lowerCAmelCase_      :					List[Any]     =			self.model_class(**A_)
									model.to(A_)
									model.eval()

									model.set_attention_slice('''auto''')
									with torch.no_grad():
											lowerCAmelCase_      :					str     =			model(**A_)
									assert output is not None

									model.set_attention_slice('''max''')
									with torch.no_grad():
											lowerCAmelCase_      :					Dict     =			model(**A_)
									assert output is not None

									model.set_attention_slice(2)
									with torch.no_grad():
											lowerCAmelCase_      :					List[str]     =			model(**A_)
									assert output is not None




							def        UpperCAmelCase__      (	self					:	Tuple):
									lowerCAmelCase_      :					List[Any]     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					Tuple     =			8

									lowerCAmelCase_      :					Any     =			self.model_class(**A_)
									model.to(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					Tuple     =			model(**A_).sample

									lowerCAmelCase_      :					str     =			create_lora_layers(A_)

									# make sure we can set a list of attention processors
									model.set_attn_processor(A_)
									model.to(A_)

									# test that attn processors can be set to itself
									model.set_attn_processor(model.attn_processors)

									with torch.no_grad():
											lowerCAmelCase_      :					Optional[Any]     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.0}).sample
											lowerCAmelCase_      :					str     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample
											lowerCAmelCase_      :					List[Any]     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample

									assert (samplea - samplea).abs().max() < 3e-3
									assert (samplea - samplea).abs().max() < 3e-3

									# sample 2 and sample 3 should be different
									assert (samplea - samplea).abs().max() > 3e-3




							def        UpperCAmelCase__      (	self					:	Any):
									lowerCAmelCase_      :					Optional[Any]     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					str     =			8

									torch.manual_seed(0)
									lowerCAmelCase_      :					Optional[int]     =			self.model_class(**A_)
									model.to(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					List[Any]     =			model(**A_).sample

									lowerCAmelCase_      :					Any     =			create_lora_layers(A_)
									model.set_attn_processor(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					Any     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample

									with tempfile.TemporaryDirectory() as tmpdirname:
											model.save_attn_procs(A_)
											self.assertTrue(os.path.isfile(os.path.join(A_    ,    '''pytorch_lora_weights.bin''')))
											torch.manual_seed(0)
											lowerCAmelCase_      :					int     =			self.model_class(**A_)
											new_model.to(A_)
											new_model.load_attn_procs(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					List[str]     =			new_model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample

									assert (sample - new_sample).abs().max() < 1e-3

									# LoRA and no LoRA should NOT be the same
									assert (sample - old_sample).abs().max() > 1e-4




							def        UpperCAmelCase__      (	self					:	Optional[Any]):
									lowerCAmelCase_      :					str     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					Optional[int]     =			8

									torch.manual_seed(0)
									lowerCAmelCase_      :					Optional[Any]     =			self.model_class(**A_)
									model.to(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					Optional[int]     =			model(**A_).sample

									lowerCAmelCase_      :					Dict     =			create_lora_layers(A_)
									model.set_attn_processor(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					str     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample

									with tempfile.TemporaryDirectory() as tmpdirname:
											model.save_attn_procs(A_    ,    safe_serialization=A_)
											self.assertTrue(os.path.isfile(os.path.join(A_    ,    '''pytorch_lora_weights.safetensors''')))
											torch.manual_seed(0)
											lowerCAmelCase_      :					List[Any]     =			self.model_class(**A_)
											new_model.to(A_)
											new_model.load_attn_procs(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					List[Any]     =			new_model(**A_    ,    cross_attention_kwargs={'''scale''': 0.5}).sample

									assert (sample - new_sample).abs().max() < 3e-3

									# LoRA and no LoRA should NOT be the same
									assert (sample - old_sample).abs().max() > 1e-4




							def        UpperCAmelCase__      (	self					:	List[str]):
									# enable deterministic behavior for gradient checkpointing
									lowerCAmelCase_      :					List[str]     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					List[str]     =			8

									torch.manual_seed(0)
									lowerCAmelCase_      :					int     =			self.model_class(**A_)
									model.to(A_)

									lowerCAmelCase_      :					Dict     =			create_lora_layers(A_    ,    mock_weights=A_)
									model.set_attn_processor(A_)
									# Saving as torch, properly reloads with directly filename
									with tempfile.TemporaryDirectory() as tmpdirname:
											model.save_attn_procs(A_)
											self.assertTrue(os.path.isfile(os.path.join(A_    ,    '''pytorch_lora_weights.bin''')))
											torch.manual_seed(0)
											lowerCAmelCase_      :					Union[str, Any]     =			self.model_class(**A_)
											new_model.to(A_)
											new_model.load_attn_procs(A_    ,    weight_name='''pytorch_lora_weights.bin''')




							def        UpperCAmelCase__      (	self					:	Union[str, Any]):
									lowerCAmelCase_      :					Tuple     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					Any     =			8

									torch.manual_seed(0)
									lowerCAmelCase_      :					Any     =			self.model_class(**A_)
									model.to(A_)

									lowerCAmelCase_      :					Dict     =			create_lora_layers(A_    ,    mock_weights=A_)
									model.set_attn_processor(A_)
									# Saving as torch, properly reloads with directly filename
									with tempfile.TemporaryDirectory() as tmpdirname:
											model.save_attn_procs(A_)
											self.assertTrue(os.path.isfile(os.path.join(A_    ,    '''pytorch_lora_weights.bin''')))
											torch.manual_seed(0)
											lowerCAmelCase_      :					Any     =			self.model_class(**A_)
											new_model.to(A_)
											with self.assertRaises(A_) as e:
													new_model.load_attn_procs(A_    ,    use_safetensors=A_)
											self.assertIn('''Error no file named pytorch_lora_weights.safetensors'''    ,    str(e.exception))




							def        UpperCAmelCase__      (	self					:	List[str]):
									lowerCAmelCase_      :					int     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					Dict     =			8

									torch.manual_seed(0)
									lowerCAmelCase_      :					List[Any]     =			self.model_class(**A_)
									model.to(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					List[str]     =			model(**A_).sample

									lowerCAmelCase_      :					Optional[int]     =			create_lora_layers(A_)
									model.set_attn_processor(A_)

									with torch.no_grad():
											lowerCAmelCase_      :					int     =			model(**A_    ,    cross_attention_kwargs={'''scale''': 0.0}).sample

									model.set_attn_processor(AttnProcessor())

									with torch.no_grad():
											lowerCAmelCase_      :					Any     =			model(**A_).sample

									assert (sample - new_sample).abs().max() < 1e-4
									assert (sample - old_sample).abs().max() < 3e-3




							@unittest.skipIf(
							    torch_device != '''cuda''' or not is_xformers_available()    ,    reason='''XFormers attention is only available with CUDA and `xformers` installed'''    ,    )
							def        UpperCAmelCase__      (	self					:	Optional[Any]):
									# enable deterministic behavior for gradient checkpointing
									lowerCAmelCase_      :					List[Any]     =			self.prepare_init_args_and_inputs_for_common()

									lowerCAmelCase_      :					Dict     =			4

									torch.manual_seed(0)
									lowerCAmelCase_      :					Tuple     =			self.model_class(**A_)
									model.to(A_)
									lowerCAmelCase_      :					List[Any]     =			create_lora_layers(A_)
									model.set_attn_processor(A_)

									# default
									with torch.no_grad():
											lowerCAmelCase_      :					Tuple     =			model(**A_).sample

											model.enable_xformers_memory_efficient_attention()
											lowerCAmelCase_      :					Union[str, Any]     =			model(**A_).sample

											model.disable_xformers_memory_efficient_attention()
											lowerCAmelCase_      :					Optional[int]     =			model(**A_).sample

									assert (sample - on_sample).abs().max() < 1e-4
									assert (sample - off_sample).abs().max() < 1e-4







							def        UpperCAmelCase__      (	self					:	Dict):
									lowerCAmelCase_      :					Optional[Any]     =			self.prepare_init_args_and_inputs_for_common()
									lowerCAmelCase_      :					Any     =			3_2

									lowerCAmelCase_      :					Optional[int]     =			self.model_class(**A_)
									model.to(A_)
									model.eval()

									with torch.no_grad():
											lowerCAmelCase_      :					Any     =			model(**A_)[0]

									model.enable_forward_chunking()
									with torch.no_grad():
											lowerCAmelCase_      :					List[str]     =			model(**A_)[0]

									self.assertEqual(output.shape    ,    output_a.shape    ,    '''Shape doesn\'t match''')
									assert np.abs(output.cpu() - output_a.cpu()).max() < 1e-2


# (todo: sayakpaul) implement SLOW tests.










import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


A__  :  Tuple       = logging.get_logger(__name__)

A__  :  List[str]       = [
    '''yjernite/retribert-base-uncased''',
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]



class     __snake_case						(					UpperCamelCase_						):
							_a      				=			RetriBertConfig
							_a      				=			None
							_a      				=			'''retribert'''




							def        UpperCAmelCase__      (	self					:	Optional[int]    ,    A_					:	Union[str, Any]):
									if isinstance(A_    ,    nn.Linear):
											module.weight.data.normal_(mean=0.0    ,    std=self.config.initializer_range)
											if module.bias is not None:
													module.bias.data.zero_()
									elif isinstance(A_    ,    nn.Embedding):
											module.weight.data.normal_(mean=0.0    ,    std=self.config.initializer_range)
											if module.padding_idx is not None:
													module.weight.data[module.padding_idx].zero_()
									elif isinstance(A_    ,    nn.LayerNorm):
											module.bias.data.zero_()
											module.weight.data.fill_(1.0)





A__  :  str       = R'''

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''



@add_start_docstrings(
    '''Bert Based model to embed queries or document for document retrieval.'''						,UpperCamelCase_						,)
class     __snake_case						(					UpperCamelCase_						):




							def __init__(	self					:	Optional[int]    ,    A_					:	RetriBertConfig):
									super().__init__(A_)
									lowerCAmelCase_      :					Optional[int]     =			config.projection_dim

									lowerCAmelCase_      :					int     =			BertModel(A_)
									lowerCAmelCase_      :					int     =			None if config.share_encoders else BertModel(A_)
									lowerCAmelCase_      :					str     =			nn.Dropout(config.hidden_dropout_prob)
									lowerCAmelCase_      :					Dict     =			nn.Linear(config.hidden_size    ,    config.projection_dim    ,    bias=A_)
									lowerCAmelCase_      :					Any     =			nn.Linear(config.hidden_size    ,    config.projection_dim    ,    bias=A_)

									lowerCAmelCase_      :					int     =			nn.CrossEntropyLoss(reduction='''mean''')

									# Initialize weights and apply final processing
									self.post_init()




							def        UpperCAmelCase__      (	self					:	str    ,    A_					:	Any    ,    A_					:	Optional[int]    ,    A_					:	Optional[int]    ,    A_					:	Optional[Any]=-1    ,    ):
									# reproduces BERT forward pass with checkpointing
									if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
											return sent_encoder(A_    ,    attention_mask=A_)[1]
									else:
											# prepare implicit variables
											lowerCAmelCase_      :					Any     =			input_ids.device
											lowerCAmelCase_      :					str     =			input_ids.size()
											lowerCAmelCase_      :					Dict     =			torch.zeros(A_    ,    dtype=torch.long    ,    device=A_)
											lowerCAmelCase_      :					List[str]     =			[None] * sent_encoder.config.num_hidden_layers
											lowerCAmelCase_      :					torch.Tensor     =			sent_encoder.get_extended_attention_mask(
											    A_    ,    A_)

											# define function for checkpointing
											def partial_encode(*A_					:	List[Any]):
													lowerCAmelCase_      :					Optional[Any]     =			sent_encoder.encoder(
													    inputs[0]    ,    attention_mask=inputs[1]    ,    head_mask=A_    ,    )
													lowerCAmelCase_      :					Any     =			encoder_outputs[0]
													lowerCAmelCase_      :					Optional[Any]     =			sent_encoder.pooler(A_)
													return pooled_output

											# run embedding layer on everything at once
											lowerCAmelCase_      :					str     =			sent_encoder.embeddings(
											    input_ids=A_    ,    position_ids=A_    ,    token_type_ids=A_    ,    inputs_embeds=A_)
											# run encoding and pooling on one mini-batch at a time
											lowerCAmelCase_      :					str     =			[]
											for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size)):
													lowerCAmelCase_      :					Optional[Any]     =			embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
													lowerCAmelCase_      :					int     =			extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
													lowerCAmelCase_      :					str     =			checkpoint.checkpoint(A_    ,    A_    ,    A_)
													pooled_output_list.append(A_)
											return torch.cat(A_    ,    dim=0)




							def        UpperCAmelCase__      (	self					:	List[str]    ,    A_					:	List[Any]    ,    A_					:	Union[str, Any]=None    ,    A_					:	Dict=-1    ,    ):
									lowerCAmelCase_      :					Optional[int]     =			self.embed_sentences_checkpointed(
									    A_    ,    A_    ,    self.bert_query    ,    A_    ,    )
									return self.project_query(A_)




							def        UpperCAmelCase__      (	self					:	Optional[Any]    ,    A_					:	str    ,    A_					:	int=None    ,    A_					:	Any=-1    ,    ):
									lowerCAmelCase_      :					Union[str, Any]     =			self.embed_sentences_checkpointed(
									    A_    ,    A_    ,    self.bert_query if self.bert_doc is None else self.bert_doc    ,    A_    ,    )
									return self.project_doc(A_)







							def        UpperCAmelCase__      (	self					:	List[Any]    ,    A_					:	torch.LongTensor    ,    A_					:	Optional[torch.FloatTensor]    ,    A_					:	torch.LongTensor    ,    A_					:	Optional[torch.FloatTensor]    ,    A_					:	int = -1    ,    ):
									lowerCAmelCase_      :					List[Any]     =			input_ids_query.device
									lowerCAmelCase_      :					List[str]     =			self.embed_questions(A_    ,    A_    ,    A_)
									lowerCAmelCase_      :					Union[str, Any]     =			self.embed_answers(A_    ,    A_    ,    A_)
									lowerCAmelCase_      :					Any     =			torch.mm(A_    ,    a_reps.t())
									lowerCAmelCase_      :					int     =			self.ce_loss(A_    ,    torch.arange(compare_scores.shape[1]).to(A_))
									lowerCAmelCase_      :					str     =			self.ce_loss(compare_scores.t()    ,    torch.arange(compare_scores.shape[0]).to(A_))
									lowerCAmelCase_      :					List[Any]     =			(loss_qa + loss_aq) / 2
									return loss





