
"""simple docstring"""




from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
 import numpy as np
 import tensorflow as tf

 from transformers import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
     TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TFSharedEmbeddings,
 )

 if _tf_gpu_memory_limit is not None:
  snake_case			       =		tf.config.list_physical_devices('GPU')
  for gpu in gpus:
   # Restrict TensorFlow to only allocate x GB of memory on the GPUs
   try:
    tf.config.set_logical_device_configuration(
        gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
    )
    snake_case			       =		tf.config.list_logical_devices('GPU')
    print('Logical GPUs', logical_gpus)
   except RuntimeError as e:
    # Virtual devices must be set before GPUs have been initialized
    print(e)




@require_tf
class 					UpperCamelCase     :

    """simple docstring"""



    UpperCAmelCase_    :				Tuple						  =    None
    UpperCAmelCase_    :				Dict						  =    ()
    UpperCAmelCase_    :				Any						  =    ()
    UpperCAmelCase_    :				Optional[Any]						  =    True
    UpperCAmelCase_    :				Optional[int]						  =    True
    UpperCAmelCase_    :				Tuple						  =    True
    UpperCAmelCase_    :				Tuple						  =    False


    def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__=False      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     copy.deepcopy(UpperCamelCase_      )

         if model_class in get_values(UpperCamelCase_      ):
              SCREAMING_SNAKE_CASE           =     {
                  k: tf.tile(tf.expand_dims(UpperCamelCase_		,     1      )		,     (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)      )
                  if isinstance(UpperCamelCase_		,     tf.Tensor      ) and v.ndim > 0
                  else v
                  for k, v in inputs_dict.items()
              }

         if return_labels:
              if model_class in get_values(UpperCamelCase_      ):
                   SCREAMING_SNAKE_CASE           =     tf.ones(self.model_tester.batch_size		,     dtype=tf.intaa      )
              elif model_class in get_values(UpperCamelCase_      ):
                   SCREAMING_SNAKE_CASE           =     tf.zeros(self.model_tester.batch_size		,     dtype=tf.intaa      )
                   SCREAMING_SNAKE_CASE           =     tf.zeros(self.model_tester.batch_size		,     dtype=tf.intaa      )
              elif model_class in [
                  *get_values(UpperCamelCase_      ),
                  *get_values(UpperCamelCase_      ),
              ]:
                   SCREAMING_SNAKE_CASE           =     tf.zeros(self.model_tester.batch_size		,     dtype=tf.intaa      )
              elif model_class in get_values(UpperCamelCase_      ):
                   SCREAMING_SNAKE_CASE           =     tf.zeros(self.model_tester.batch_size		,     dtype=tf.intaa      )
              elif model_class in [
                  *get_values(UpperCamelCase_      ),
                  *get_values(UpperCamelCase_      ),
                  *get_values(UpperCamelCase_      ),
                  *get_values(UpperCamelCase_      ),
                  *get_values(UpperCamelCase_      ),
              ]:
                   SCREAMING_SNAKE_CASE           =     tf.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length)		,     dtype=tf.intaa      )
         return inputs_dict


    @slow
    def 							A (    self      )   ->   Tuple:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes[:2]:
              SCREAMING_SNAKE_CASE           =     self._prepare_for_class(UpperCamelCase_		,     UpperCamelCase_      )
              SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )

              @tf.function
              def run_in_graph_mode():
                   return model(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     run_in_graph_mode()
              self.assertIsNotNone(UpperCamelCase_      )


    @slow
    def 							A (    self      )   ->   Optional[Any]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes[:2]:
              SCREAMING_SNAKE_CASE           =     self._prepare_for_class(UpperCamelCase_		,     UpperCamelCase_      )
              SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )

              @tf.function(experimental_compile=UpperCamelCase_      )
              def run_in_graph_mode():
                   return model(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     run_in_graph_mode()
              self.assertIsNotNone(UpperCamelCase_      )


    @slow
    def 							A (    self      )   ->   Any:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes[:2]:
              SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )
              if getattr(UpperCamelCase_		,     'hf_compute_loss'		,     UpperCamelCase_      ):
                   # Test that model correctly compute the loss with kwargs
                   SCREAMING_SNAKE_CASE           =     self._prepare_for_class(inputs_dict.copy()		,     UpperCamelCase_		,     return_labels=UpperCamelCase_      )
                   # Is there a better way to remove these decoder inputs?
                   SCREAMING_SNAKE_CASE           =     {
                       key: val
                       for key, val in prepared_for_class.items()
                       if key not in ('head_mask', 'decoder_head_mask', 'cross_attn_head_mask', 'decoder_input_ids')
                   }

                   SCREAMING_SNAKE_CASE           =     {
                       'labels',
                       'label',
                       'label_ids',
                       'start_positions',
                       'start_position',
                       'end_positions',
                       'end_position',
                       'next_sentence_label',
                   }
                   SCREAMING_SNAKE_CASE           =     possible_label_cols.intersection(set(UpperCamelCase_      )      )
                   self.assertGreater(len(UpperCamelCase_      )		,     0		,     msg='No matching label names found!'      )
                   SCREAMING_SNAKE_CASE           =     {key: val for key, val in prepared_for_class.items() if key in label_names}
                   SCREAMING_SNAKE_CASE           =     {key: val for key, val in prepared_for_class.items() if key not in label_names}
                   self.assertGreater(len(UpperCamelCase_      )		,     0      )

                   # Make sure it works with XLA!
                   model.compile(optimizer=tf.keras.optimizers.SGD(0.0      )		,     jit_compile=UpperCamelCase_      )
                   # Make sure the model fits without crashing regardless of where we pass the labels
                   SCREAMING_SNAKE_CASE           =     model.fit(
                       UpperCamelCase_		,     validation_data=UpperCamelCase_		,     steps_per_epoch=1		,     validation_steps=1		,     shuffle=UpperCamelCase_		,     verbose=0		,     )
                   SCREAMING_SNAKE_CASE           =     history.history['loss'][0]
                   self.assertTrue(not isnan(UpperCamelCase_      )      )
                   SCREAMING_SNAKE_CASE           =     history.history['val_loss'][0]
                   self.assertTrue(not isnan(UpperCamelCase_      )      )

                   # Now test it with separate labels, to make sure that path works in XLA too.
                   SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )
                   model.compile(optimizer=tf.keras.optimizers.SGD(0.0      )		,     jit_compile=UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     model.fit(
                       UpperCamelCase_		,     UpperCamelCase_		,     validation_data=(inputs_minus_labels, labels)		,     steps_per_epoch=1		,     validation_steps=1		,     shuffle=UpperCamelCase_		,     verbose=0		,     )

                   SCREAMING_SNAKE_CASE           =     history.history['loss'][0]
                   self.assertTrue(not isnan(UpperCamelCase_      )      )
                   SCREAMING_SNAKE_CASE           =     history.history['val_loss'][0]
                   self.assertTrue(not isnan(UpperCamelCase_      )      )


    @slow
    def 							A (    self      )   ->   Dict:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
         SCREAMING_SNAKE_CASE           =     True
         SCREAMING_SNAKE_CASE           =     True

         if hasattr(UpperCamelCase_		,     'use_cache'      ):
              SCREAMING_SNAKE_CASE           =     True

         SCREAMING_SNAKE_CASE           =     getattr(self.model_tester		,     'encoder_seq_length'		,     self.model_tester.seq_length      )
         SCREAMING_SNAKE_CASE           =     getattr(self.model_tester		,     'key_length'		,     UpperCamelCase_      )

         for model_class in self.all_model_classes[:2]:
              SCREAMING_SNAKE_CASE           =     self._prepare_for_class(UpperCamelCase_		,     UpperCamelCase_      )
              SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )
              model.build()
              SCREAMING_SNAKE_CASE           =     len(model(UpperCamelCase_      )      )

              for key in list(class_inputs_dict.keys()      ):
                   # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
                   if key not in model.input_signature:
                        del class_inputs_dict[key]
                   # Check it's a tensor, in case the inputs dict has some bools in it too
                   elif isinstance(class_inputs_dict[key]		,     tf.Tensor      ) and class_inputs_dict[key].dtype.is_integer:
                        SCREAMING_SNAKE_CASE           =     tf.cast(class_inputs_dict[key]		,     tf.intaa      )

              if set(class_inputs_dict.keys()      ) != set(model.input_signature.keys()      ):
                   continue  # Some models have inputs that the preparation functions don't create, we skip those

              with tempfile.TemporaryDirectory() as tmpdirname:
                   model.save_pretrained(UpperCamelCase_		,     saved_model=UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     os.path.join(UpperCamelCase_		,     'saved_model'		,     '1'      )
                   SCREAMING_SNAKE_CASE           =     tf.keras.models.load_model(UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     model(UpperCamelCase_      )

                   if self.is_encoder_decoder:
                        SCREAMING_SNAKE_CASE           =     outputs['encoder_hidden_states']
                        SCREAMING_SNAKE_CASE           =     outputs['encoder_attentions']
                   else:
                        SCREAMING_SNAKE_CASE           =     outputs['hidden_states']
                        SCREAMING_SNAKE_CASE           =     outputs['attentions']

                   self.assertEqual(len(UpperCamelCase_      )		,     UpperCamelCase_      )

                   SCREAMING_SNAKE_CASE           =     getattr(
                       self.model_tester		,     'expected_num_hidden_layers'		,     self.model_tester.num_hidden_layers + 1      )

                   self.assertEqual(len(UpperCamelCase_      )		,     UpperCamelCase_      )
                   self.assertListEqual(
                       list(output_hidden_states[0].shape[-2:]      )		,     [self.model_tester.seq_length, self.model_tester.hidden_size]		,     )

                   self.assertEqual(len(UpperCamelCase_      )		,     self.model_tester.num_hidden_layers      )
                   self.assertListEqual(
                       list(output_attentions[0].shape[-3:]      )		,     [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]		,     )


    @slow
    def 							A (    self      )   ->   Any:
         """simple docstring"""







         tf.keras.mixed_precision.set_global_policy('mixed_float16'      )

         # try/finally block to ensure subsequent tests run in float32
         try:
              SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
              for model_class in self.all_model_classes[:2]:
                   SCREAMING_SNAKE_CASE           =     self._prepare_for_class(UpperCamelCase_		,     UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     model(UpperCamelCase_      )

                   self.assertIsNotNone(UpperCamelCase_      )
         finally:
              tf.keras.mixed_precision.set_global_policy('float32'      )


    @slow
    def 							A (    self      )   ->   Optional[Any]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()
         # head_mask and decoder_head_mask has different shapes than other input args
         if "head_mask" in inputs_dict:
              del inputs_dict["head_mask"]
         if "decoder_head_mask" in inputs_dict:
              del inputs_dict["decoder_head_mask"]
         if "cross_attn_head_mask" in inputs_dict:
              del inputs_dict["cross_attn_head_mask"]
         SCREAMING_SNAKE_CASE           =     {
             module_member
             for model_class in self.all_model_classes
             for module in (import_module(model_class.__module__      ),)
             for module_member_name in dir(UpperCamelCase_      )
             if module_member_name.endswith('MainLayer'      )
             for module_member in (getattr(UpperCamelCase_		,     UpperCamelCase_      ),)
             if isinstance(UpperCamelCase_		,     UpperCamelCase_      )
             and tf.keras.layers.Layer in module_member.__bases__
             and getattr(UpperCamelCase_		,     '_keras_serializable'		,     UpperCamelCase_      )
         }

         for main_layer_class in tf_main_layer_classes:
              # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
              if "T5" in main_layer_class.__name__:
                   # Take the same values than in TFT5ModelTester for this shared layer
                   SCREAMING_SNAKE_CASE           =     TFSharedEmbeddings(self.model_tester.vocab_size		,     self.model_tester.hidden_size		,     name='shared'      )
                   SCREAMING_SNAKE_CASE           =     False
                   SCREAMING_SNAKE_CASE           =     main_layer_class(UpperCamelCase_		,     embed_tokens=UpperCamelCase_      )
              else:
                   SCREAMING_SNAKE_CASE           =     main_layer_class(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     {
                  name: tf.keras.Input(tensor.shape[1:]		,     dtype=tensor.dtype      ) for name, tensor in inputs_dict.items()
              }

              if hasattr(self.model_tester		,     'num_labels'      ):
                   SCREAMING_SNAKE_CASE           =     self.model_tester.num_labels
              else:
                   SCREAMING_SNAKE_CASE           =     2

              SCREAMING_SNAKE_CASE           =     tf.data.Dataset.from_tensor_slices(
                  (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)      ))      ).batch(1      )

              SCREAMING_SNAKE_CASE           =     main_layer(UpperCamelCase_      )[0]
              SCREAMING_SNAKE_CASE           =     tf.keras.layers.Dense(UpperCamelCase_		,     activation='softmax'		,     name='outputs'      )(UpperCamelCase_      )
              SCREAMING_SNAKE_CASE           =     tf.keras.models.Model(inputs=UpperCamelCase_		,     outputs=[outputs]      )

              model.compile(loss='binary_crossentropy'		,     optimizer='adam'		,     metrics=['binary_accuracy']      )
              model.fit(UpperCamelCase_		,     epochs=1      )

              with tempfile.TemporaryDirectory() as tmpdirname:
                   SCREAMING_SNAKE_CASE           =     os.path.join(UpperCamelCase_		,     'keras_model.h5'      )
                   model.save(UpperCamelCase_      )
                   if "T5" in main_layer_class.__name__:
                        SCREAMING_SNAKE_CASE           =     tf.keras.models.load_model(
                            UpperCamelCase_		,     custom_objects={
                                main_layer_class.__name__: main_layer_class,
                                'TFSharedEmbeddings': TFSharedEmbeddings,
                            }		,     )
                   else:
                        SCREAMING_SNAKE_CASE           =     tf.keras.models.load_model(
                            UpperCamelCase_		,     custom_objects={main_layer_class.__name__: main_layer_class}      )
                   assert isinstance(UpperCamelCase_		,     tf.keras.Model      )
                   model(UpperCamelCase_      )


    @slow
    def 							A (    self      )   ->   List[Any]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes[:2]:
              SCREAMING_SNAKE_CASE           =     model_class(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     copy.deepcopy(UpperCamelCase_      )

              if not self.is_encoder_decoder:
                   SCREAMING_SNAKE_CASE           =     inputs['input_ids']
                   del inputs["input_ids"]
              else:
                   SCREAMING_SNAKE_CASE           =     inputs['input_ids']
                   SCREAMING_SNAKE_CASE           =     inputs.get('decoder_input_ids'		,     UpperCamelCase_      )
                   del inputs["input_ids"]
                   inputs.pop('decoder_input_ids'		,     UpperCamelCase_      )

              if not self.is_encoder_decoder:
                   SCREAMING_SNAKE_CASE           =     model.get_input_embeddings()(UpperCamelCase_      )
              else:
                   SCREAMING_SNAKE_CASE           =     model.get_input_embeddings()(UpperCamelCase_      )
                   SCREAMING_SNAKE_CASE           =     model.get_input_embeddings()(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     self._prepare_for_class(UpperCamelCase_		,     UpperCamelCase_      )

              @tf.function
              def run_in_graph_mode():
                   return model(UpperCamelCase_      )

              SCREAMING_SNAKE_CASE           =     run_in_graph_mode()
              self.assertIsNotNone(UpperCamelCase_      )


    def 							A (    self		,     lowercase__		,     lowercase__      )   ->   Tuple:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     []
         if model.config.bos_token_id is not None:
              special_tokens.append(model.config.bos_token_id      )
         if model.config.pad_token_id is not None:
              special_tokens.append(model.config.pad_token_id      )
         if model.config.eos_token_id is not None:
              special_tokens.append(model.config.eos_token_id      )

         # create random bad tokens that are not special tokens
         SCREAMING_SNAKE_CASE           =     []
         while len(UpperCamelCase_      ) < num_bad_tokens:
              SCREAMING_SNAKE_CASE           =     tf.squeeze(ids_tensor((1, 1)		,     self.model_tester.vocab_size      )		,     0      ).numpy()[0]
              if token not in special_tokens:
                   bad_tokens.append(UpperCamelCase_      )
         return bad_tokens


    def 							A (    self		,     lowercase__      )   ->   List[Any]:
         """simple docstring"""







         for token_id in output_ids[0].numpy().tolist():
              self.assertGreaterEqual(UpperCamelCase_		,     0      )
              self.assertLess(UpperCamelCase_		,     self.model_tester.vocab_size      )




    def 							A (    self		,     lowercase__		,     lowercase__      )   ->   Any:
         """simple docstring"""







         for bad_word_ids in bad_words_ids:
              # for all slices in batch
              for generated_ids_slice in generated_ids:
                   # for all word idx
                   for i in range(len(UpperCamelCase_      )		,     len(UpperCamelCase_      )      ):
                        # if tokens match
                        if generated_ids_slice[i - len(UpperCamelCase_      ) : i] == bad_word_ids:
                             return True
         return False


"""simple docstring"""




import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


snake_case			       =		'block_records.npy'


snake_case			       =		logging.get_logger(__name__)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					import tensorflow.compat.va as tf

					SCREAMING_SNAKE_CASE           =     tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE_,					buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4      )
					SCREAMING_SNAKE_CASE           =     blocks_dataset.batch(SCREAMING_SNAKE_CASE_,					drop_remainder=SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     next(blocks_dataset.take(1      ).as_numpy_iterator()      )

					return np_record




class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__		,     lowercase__=2		,     lowercase__=1000		,     lowercase__=100		,     lowercase__=100000		,     )   ->   Dict:
									"""simple docstring"""







									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									SCREAMING_SNAKE_CASE           =     Builder(db=lowercase__		,     num_neighbors=lowercase__		,     distance_measure='dot_product'      )
									SCREAMING_SNAKE_CASE           =     builder.tree(
									    num_leaves=lowercase__		,     num_leaves_to_search=lowercase__		,     training_sample_size=lowercase__      )
									SCREAMING_SNAKE_CASE           =     builder.score_ah(dimensions_per_block=lowercase__      )

									SCREAMING_SNAKE_CASE           =     builder.build()




				def 							A (    self		,     lowercase__      )   ->   str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.searcher.search_batched(question_projection.detach().cpu()      )
									return retrieved_block_ids.astype('int64'      )






class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__      )   ->   Dict:
									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE           =     block_records
									SCREAMING_SNAKE_CASE           =     tokenizer


				def __call__(    self		,     lowercase__		,     lowercase__		,     lowercase__		,     lowercase__=None		,     lowercase__="pt"      )   ->   Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     np.take(self.block_records		,     indices=lowercase__		,     axis=0      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer.decode(question_input_ids[0]		,     skip_special_tokens=lowercase__      )

									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									for retrieved_block in retrieved_blocks:
														text.append(lowercase__      )
														text_pair.append(retrieved_block.decode()      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer(
									    lowercase__		,     lowercase__		,     padding=lowercase__		,     truncation=lowercase__		,     return_special_tokens_mask=lowercase__		,     max_length=lowercase__      )
									SCREAMING_SNAKE_CASE           =     concat_inputs.convert_to_tensors(lowercase__      )

									if answer_ids is not None:
														return self.block_has_answer(lowercase__		,     lowercase__      ) + (concat_inputs_tensors,)
									else:
														return (None, None, None, concat_inputs_tensors)


				@classmethod
				def 							A (    cls		,     lowercase__		,     *lowercase__		,     **lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									if os.path.isdir(lowercase__      ):
														SCREAMING_SNAKE_CASE           =     os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )
									else:
														SCREAMING_SNAKE_CASE           =     hf_hub_download(
														    repo_id=lowercase__		,     filename=_REALM_BLOCK_RECORDS_FILENAME		,     **lowercase__      )
									SCREAMING_SNAKE_CASE           =     np.load(lowercase__		,     allow_pickle=lowercase__      )

									SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained(lowercase__		,     *lowercase__		,     **lowercase__      )

									return cls(lowercase__		,     lowercase__      )


				def 							A (    self		,     lowercase__      )   ->   List[str]:
									"""simple docstring"""







									np.save(os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )		,     self.block_records      )
									# save tokenizer
									self.tokenizer.save_pretrained(lowercase__      )




				def 							A (    self		,     lowercase__		,     lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     0

									for input_id in concat_inputs.input_ids:
														SCREAMING_SNAKE_CASE           =     input_id.tolist()
														# Check answers between two [SEP] tokens
														SCREAMING_SNAKE_CASE           =     input_id_list.index(self.tokenizer.sep_token_id      )
														SCREAMING_SNAKE_CASE           =     first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

														start_pos.append([]      )
														end_pos.append([]      )
														for answer in answer_ids:
																			for idx in range(first_sep_idx + 1		,     lowercase__      ):
																								if answer[0] == input_id_list[idx]:
																													if input_id_list[idx : idx + len(lowercase__      )] == answer:
																																		start_pos[-1].append(lowercase__      )
																																		end_pos[-1].append(idx + len(lowercase__      ) - 1      )

														if len(start_pos[-1]      ) == 0:
																			has_answers.append(lowercase__      )
														else:
																			has_answers.append(lowercase__      )
																			if len(start_pos[-1]      ) > max_answers:
																								SCREAMING_SNAKE_CASE           =     len(start_pos[-1]      )

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(lowercase__		,     lowercase__      ):
														if len(start_pos_      ) < max_answers:
																			SCREAMING_SNAKE_CASE           =     [-1] * (max_answers - len(start_pos_      ))
																			start_pos_ += padded
																			end_pos_ += padded
									return has_answers, start_pos, end_pos
