



import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


A__     				=	logging.getLogger(__name__)





class  __lowerCAmelCase			:




				def __init__(							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   False




				def 							snake_case       (							self						,    _snake_case						,    _snake_case						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									if not self.initialized:
														_lowerCAmelCase      =   RagRetriever(
														    lowerCAmelCase__						,    question_encoder_tokenizer=lowerCAmelCase__						,    generator_tokenizer=lowerCAmelCase__						,    index=lowerCAmelCase__						,    init_retrieval=lowerCAmelCase__						,    )
														_lowerCAmelCase      =   True




				def 							snake_case       (							self		):



									"""simple docstring"""





									self.retriever.index.init_index()




				def 							snake_case       (							self		):



									"""simple docstring"""





									del self.retriever
									_lowerCAmelCase      =   False




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase     ,     _lowerCAmelCase      =   self.retriever._main_retrieve(lowerCAmelCase__						,    lowerCAmelCase__		)
									_lowerCAmelCase      =   self.retriever.index.get_doc_dicts(lowerCAmelCase__		)
									return doc_ids, retrieved_doc_embeds, doc_dicts





class  __lowerCAmelCase			(					_UpperCAmelCase	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case						,    _snake_case						,    _snake_case=None		):



									"""simple docstring"""





									if index is not None and index.is_initialized() and len(lowerCAmelCase__		) > 0:
														raise ValueError(
														    """When using Ray for distributed fine-tuning, """
														    """you\'ll need to provide the paths instead, """
														    """as the dataset and the index are loaded """
														    """separately. More info in examples/rag/use_own_knowledge_dataset.py """		)

									super().__init__(
									    lowerCAmelCase__						,    question_encoder_tokenizer=lowerCAmelCase__						,    generator_tokenizer=lowerCAmelCase__						,    index=lowerCAmelCase__						,    init_retrieval=lowerCAmelCase__						,    )

									_lowerCAmelCase      =   retrieval_workers
									_lowerCAmelCase      =   question_encoder_tokenizer
									_lowerCAmelCase      =   generator_tokenizer
									if len(self.retrieval_workers		) > 0:
														ray.get(
														    [
														        worker.create_rag_retriever.remote(lowerCAmelCase__						,    lowerCAmelCase__						,    lowerCAmelCase__						,    lowerCAmelCase__		)
														        for worker in self.retrieval_workers
														    ]		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									logger.info("""initializing retrieval"""		)

									if len(self.retrieval_workers		) > 0:
														ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]		)
									else:
														# Non-distributed training. Load index into this same process.
														self.index.init_index()




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									if len(self.retrieval_workers		) > 0:
														# Select a random retrieval actor.
														_lowerCAmelCase      =   self.retrieval_workers[random.randint(0						,    len(self.retrieval_workers		) - 1		)]
														_lowerCAmelCase     ,     _lowerCAmelCase     ,     _lowerCAmelCase      =   ray.get(
														    random_worker.retrieve.remote(lowerCAmelCase__						,    lowerCAmelCase__		)		)
									else:
														_lowerCAmelCase     ,     _lowerCAmelCase      =   self._main_retrieve(lowerCAmelCase__						,    lowerCAmelCase__		)
														_lowerCAmelCase      =   self.index.get_doc_dicts(lowerCAmelCase__		)
									return retrieved_doc_embeds, doc_ids, doc_dicts




				@classmethod
				def 							snake_case       (							cls						,    _snake_case						,    _snake_case=None						,    **_snake_case		):



									"""simple docstring"""





									return super(lowerCAmelCase__						,    cls		).get_tokenizers(lowerCAmelCase__						,    lowerCAmelCase__						,    **lowerCAmelCase__		)




				@classmethod
				def 							snake_case       (							cls						,    _snake_case						,    _snake_case						,    _snake_case=None						,    **_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   kwargs.pop("""config"""						,    lowerCAmelCase__		) or RagConfig.from_pretrained(lowerCAmelCase__						,    **lowerCAmelCase__		)
									_lowerCAmelCase      =   RagTokenizer.from_pretrained(lowerCAmelCase__						,    config=lowerCAmelCase__		)
									_lowerCAmelCase      =   rag_tokenizer.question_encoder
									_lowerCAmelCase      =   rag_tokenizer.generator

									if indexed_dataset is not None:
														_lowerCAmelCase      =   """custom"""
														_lowerCAmelCase      =   CustomHFIndex(config.retrieval_vector_size						,    lowerCAmelCase__		)
									else:
														_lowerCAmelCase      =   cls._build_index(lowerCAmelCase__		)

									return cls(
									    lowerCAmelCase__						,    question_encoder_tokenizer=lowerCAmelCase__						,    generator_tokenizer=lowerCAmelCase__						,    retrieval_workers=lowerCAmelCase__						,    index=lowerCAmelCase__						,    )




				def 							snake_case       (							self		):



									"""simple docstring"""





									logger.info("""re-loading the new dataset with embeddings"""		)
									# access from the training loop

									ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]		)

									# build the index object again
									_lowerCAmelCase      =   self._build_index(self.config		)

									ray.get(
									    [
									        worker.create_rag_retriever.remote(
									            self.config						,    self.question_encoder_tokenizer						,    self.generator_tokenizer						,    lowerCAmelCase__		)
									        for worker in self.retrieval_workers
									    ]		)








import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
	from .testing_utils import (
	    floats_tensor,
	    load_hf_numpy,
	    load_image,
	    load_numpy,
	    load_pt,
	    nightly,
	    parse_flag_from_env,
	    print_tensor_test,
	    require_torch_a,
	    require_torch_gpu,
	    skip_mps,
	    slow,
	    torch_all_close,
	    torch_device,
	)
	from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


A__     				=	get_logger(__name__)


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					if version.parse(snake_case      ) < version.parse(snake_case      ):
										if "dev" in min_version:
															_lowerCAmelCase      =   (
															    """This example requires a source install from HuggingFace diffusers (see """
															    """`https://huggingface.co/docs/diffusers/installation#install-from-source`),"""
															)
										else:
															_lowerCAmelCase      =   F'This example requires a minimum version of {min_version},'
										error_message += F' but the version found is {__version__}.\n'
										raise ImportError(snake_case      )



