
"""simple docstring"""




import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
 from .testing_utils import (
     floats_tensor,
     load_hf_numpy,
     load_image,
     load_numpy,
     load_pt,
     nightly,
     parse_flag_from_env,
     print_tensor_test,
     require_torch_a,
     require_torch_gpu,
     skip_mps,
     slow,
     torch_all_close,
     torch_device,
 )
 from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


snake_case			       =		get_logger(__name__)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
     if version.parse(SCREAMING_SNAKE_CASE_      ) < version.parse(SCREAMING_SNAKE_CASE_      ):
          if "dev" in min_version:
               SCREAMING_SNAKE_CASE           =     (
                   'This example requires a source install from HuggingFace diffusers (see '
                   '`https://huggingface.co/docs/diffusers/installation#install-from-source`),'
               )
          else:
               SCREAMING_SNAKE_CASE           =     f'''This example requires a minimum version of {min_version},'''
          error_message += f''' but the version found is {__version__}.\n'''
          raise ImportError(SCREAMING_SNAKE_CASE_      )


"""simple docstring"""




import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


snake_case			       =		'block_records.npy'


snake_case			       =		logging.get_logger(__name__)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					import tensorflow.compat.va as tf

					SCREAMING_SNAKE_CASE           =     tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE_,					buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4      )
					SCREAMING_SNAKE_CASE           =     blocks_dataset.batch(SCREAMING_SNAKE_CASE_,					drop_remainder=SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     next(blocks_dataset.take(1      ).as_numpy_iterator()      )

					return np_record




class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__		,     lowercase__=2		,     lowercase__=1000		,     lowercase__=100		,     lowercase__=100000		,     )   ->   Dict:
									"""simple docstring"""







									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									SCREAMING_SNAKE_CASE           =     Builder(db=lowercase__		,     num_neighbors=lowercase__		,     distance_measure='dot_product'      )
									SCREAMING_SNAKE_CASE           =     builder.tree(
									    num_leaves=lowercase__		,     num_leaves_to_search=lowercase__		,     training_sample_size=lowercase__      )
									SCREAMING_SNAKE_CASE           =     builder.score_ah(dimensions_per_block=lowercase__      )

									SCREAMING_SNAKE_CASE           =     builder.build()




				def 							A (    self		,     lowercase__      )   ->   str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.searcher.search_batched(question_projection.detach().cpu()      )
									return retrieved_block_ids.astype('int64'      )






class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__      )   ->   Dict:
									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE           =     block_records
									SCREAMING_SNAKE_CASE           =     tokenizer


				def __call__(    self		,     lowercase__		,     lowercase__		,     lowercase__		,     lowercase__=None		,     lowercase__="pt"      )   ->   Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     np.take(self.block_records		,     indices=lowercase__		,     axis=0      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer.decode(question_input_ids[0]		,     skip_special_tokens=lowercase__      )

									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									for retrieved_block in retrieved_blocks:
														text.append(lowercase__      )
														text_pair.append(retrieved_block.decode()      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer(
									    lowercase__		,     lowercase__		,     padding=lowercase__		,     truncation=lowercase__		,     return_special_tokens_mask=lowercase__		,     max_length=lowercase__      )
									SCREAMING_SNAKE_CASE           =     concat_inputs.convert_to_tensors(lowercase__      )

									if answer_ids is not None:
														return self.block_has_answer(lowercase__		,     lowercase__      ) + (concat_inputs_tensors,)
									else:
														return (None, None, None, concat_inputs_tensors)


				@classmethod
				def 							A (    cls		,     lowercase__		,     *lowercase__		,     **lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									if os.path.isdir(lowercase__      ):
														SCREAMING_SNAKE_CASE           =     os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )
									else:
														SCREAMING_SNAKE_CASE           =     hf_hub_download(
														    repo_id=lowercase__		,     filename=_REALM_BLOCK_RECORDS_FILENAME		,     **lowercase__      )
									SCREAMING_SNAKE_CASE           =     np.load(lowercase__		,     allow_pickle=lowercase__      )

									SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained(lowercase__		,     *lowercase__		,     **lowercase__      )

									return cls(lowercase__		,     lowercase__      )


				def 							A (    self		,     lowercase__      )   ->   List[str]:
									"""simple docstring"""







									np.save(os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )		,     self.block_records      )
									# save tokenizer
									self.tokenizer.save_pretrained(lowercase__      )




				def 							A (    self		,     lowercase__		,     lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     0

									for input_id in concat_inputs.input_ids:
														SCREAMING_SNAKE_CASE           =     input_id.tolist()
														# Check answers between two [SEP] tokens
														SCREAMING_SNAKE_CASE           =     input_id_list.index(self.tokenizer.sep_token_id      )
														SCREAMING_SNAKE_CASE           =     first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

														start_pos.append([]      )
														end_pos.append([]      )
														for answer in answer_ids:
																			for idx in range(first_sep_idx + 1		,     lowercase__      ):
																								if answer[0] == input_id_list[idx]:
																													if input_id_list[idx : idx + len(lowercase__      )] == answer:
																																		start_pos[-1].append(lowercase__      )
																																		end_pos[-1].append(idx + len(lowercase__      ) - 1      )

														if len(start_pos[-1]      ) == 0:
																			has_answers.append(lowercase__      )
														else:
																			has_answers.append(lowercase__      )
																			if len(start_pos[-1]      ) > max_answers:
																								SCREAMING_SNAKE_CASE           =     len(start_pos[-1]      )

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(lowercase__		,     lowercase__      ):
														if len(start_pos_      ) < max_answers:
																			SCREAMING_SNAKE_CASE           =     [-1] * (max_answers - len(start_pos_      ))
																			start_pos_ += padded
																			end_pos_ += padded
									return has_answers, start_pos, end_pos
