
import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


SCREAMING_SNAKE_CASE__							=    logging.getLogger(__name__)
class      A__       :





							def __init__(			self  :       Dict    )						->		Dict:



											"""simple docstring"""


											__lowercase								= False





							def       a__			(			self  :       int					,			_UpperCAmelCase  :       str					,			_UpperCAmelCase  :       str					,			_UpperCAmelCase  :       List[Any]					,			_UpperCAmelCase  :       Tuple    )						->		Dict:



											"""simple docstring"""


											if not self.initialized:
															__lowercase								= RagRetriever(
															    _UpperCAmelCase					,			question_encoder_tokenizer=_UpperCAmelCase					,			generator_tokenizer=_UpperCAmelCase					,			index=_UpperCAmelCase					,			init_retrieval=_UpperCAmelCase					,			)
															__lowercase								= True





							def       a__			(			self  :       Dict    )						->		Any:



											"""simple docstring"""


											self.retriever.index.init_index()





							def       a__			(			self  :       Tuple    )						->		Optional[Any]:



											"""simple docstring"""


											del self.retriever
											__lowercase								= False





							def       a__			(			self  :       List[str]					,			_UpperCAmelCase  :       Optional[int]					,			_UpperCAmelCase  :       str    )						->		str:



											"""simple docstring"""


											__lowercase		, __lowercase								= self.retriever._main_retrieve(_UpperCAmelCase					,			_UpperCAmelCase    )
											__lowercase								= self.retriever.index.get_doc_dicts(_UpperCAmelCase    )
											return doc_ids, retrieved_doc_embeds, doc_dicts






class      A__       (						lowerCAmelCase__    ):





							def __init__(			self  :       str					,			_UpperCAmelCase  :       str					,			_UpperCAmelCase  :       Dict					,			_UpperCAmelCase  :       List[str]					,			_UpperCAmelCase  :       int					,			_UpperCAmelCase  :       List[Any]=None    )						->		int:



											"""simple docstring"""


											if index is not None and index.is_initialized() and len(_UpperCAmelCase    ) > 0:
															raise ValueError(
															    'When using Ray for distributed fine-tuning, '
															    'you\'ll need to provide the paths instead, '
															    'as the dataset and the index are loaded '
															    'separately. More info in examples/rag/use_own_knowledge_dataset.py '    )

											super().__init__(
											    _UpperCAmelCase					,			question_encoder_tokenizer=_UpperCAmelCase					,			generator_tokenizer=_UpperCAmelCase					,			index=_UpperCAmelCase					,			init_retrieval=_UpperCAmelCase					,			)

											__lowercase								= retrieval_workers
											__lowercase								= question_encoder_tokenizer
											__lowercase								= generator_tokenizer
											if len(self.retrieval_workers    ) > 0:
															ray.get(
															    [
															        worker.create_rag_retriever.remote(_UpperCAmelCase					,			_UpperCAmelCase					,			_UpperCAmelCase					,			_UpperCAmelCase    )
															        for worker in self.retrieval_workers
															    ]    )





							def       a__			(			self  :       Optional[int]    )						->		Optional[Any]:



											"""simple docstring"""


											logger.info('initializing retrieval'    )

											if len(self.retrieval_workers    ) > 0:
															ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]    )
											else:
															# Non-distributed training. Load index into this same process.
															self.index.init_index()





							def       a__			(			self  :       Optional[int]					,			_UpperCAmelCase  :       int					,			_UpperCAmelCase  :       int    )						->		str:



											"""simple docstring"""


											if len(self.retrieval_workers    ) > 0:
															# Select a random retrieval actor.
															__lowercase								= self.retrieval_workers[random.randint(0					,			len(self.retrieval_workers    ) - 1    )]
															__lowercase		, __lowercase		, __lowercase								= ray.get(
															    random_worker.retrieve.remote(_UpperCAmelCase					,			_UpperCAmelCase    )    )
											else:
															__lowercase		, __lowercase								= self._main_retrieve(_UpperCAmelCase					,			_UpperCAmelCase    )
															__lowercase								= self.index.get_doc_dicts(_UpperCAmelCase    )
											return retrieved_doc_embeds, doc_ids, doc_dicts





							@classmethod
							def       a__			(			cls  :       List[str]					,			_UpperCAmelCase  :       Tuple					,			_UpperCAmelCase  :       Optional[Any]=None					,			**_UpperCAmelCase  :       Dict    )						->		Dict:



											"""simple docstring"""


											return super(_UpperCAmelCase					,			cls    ).get_tokenizers(_UpperCAmelCase					,			_UpperCAmelCase					,			**_UpperCAmelCase    )





							@classmethod
							def       a__			(			cls  :       Optional[Any]					,			_UpperCAmelCase  :       str					,			_UpperCAmelCase  :       Tuple					,			_UpperCAmelCase  :       Optional[int]=None					,			**_UpperCAmelCase  :       int    )						->		int:



											"""simple docstring"""


											__lowercase								= kwargs.pop('config'					,			_UpperCAmelCase    ) or RagConfig.from_pretrained(_UpperCAmelCase					,			**_UpperCAmelCase    )
											__lowercase								= RagTokenizer.from_pretrained(_UpperCAmelCase					,			config=_UpperCAmelCase    )
											__lowercase								= rag_tokenizer.question_encoder
											__lowercase								= rag_tokenizer.generator

											if indexed_dataset is not None:
															__lowercase								= 'custom'
															__lowercase								= CustomHFIndex(config.retrieval_vector_size					,			_UpperCAmelCase    )
											else:
															__lowercase								= cls._build_index(_UpperCAmelCase    )

											return cls(
											    _UpperCAmelCase					,			question_encoder_tokenizer=_UpperCAmelCase					,			generator_tokenizer=_UpperCAmelCase					,			retrieval_workers=_UpperCAmelCase					,			index=_UpperCAmelCase					,			)





							def       a__			(			self  :       List[Any]    )						->		str:



											"""simple docstring"""


											logger.info('re-loading the new dataset with embeddings'    )
											# access from the training loop

											ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]    )

											# build the index object again
											__lowercase								= self._build_index(self.config    )

											ray.get(
											    [
											        worker.create_rag_retriever.remote(
											            self.config					,			self.question_encoder_tokenizer					,			self.generator_tokenizer					,			_UpperCAmelCase    )
											        for worker in self.retrieval_workers
											    ]    )







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
							SCREAMING_SNAKE_CASE__							=    None


SCREAMING_SNAKE_CASE__							=    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__							=    {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__							=    {
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__							=    {
    """xlm-roberta-base""": 512,
    """xlm-roberta-large""": 512,
    """xlm-roberta-large-finetuned-conll02-dutch""": 512,
    """xlm-roberta-large-finetuned-conll02-spanish""": 512,
    """xlm-roberta-large-finetuned-conll03-english""": 512,
    """xlm-roberta-large-finetuned-conll03-german""": 512,
}
class      A__       (						lowerCAmelCase__    ):
							lowerCAmelCase__		:						Dict         =							VOCAB_FILES_NAMES
							lowerCAmelCase__		:						List[str]         =							PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__		:						List[Any]         =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase__		:						Dict         =							["input_ids", "attention_mask"]
							lowerCAmelCase__		:						List[str]         =							XLMRobertaTokenizer





							def __init__(			self  :       Tuple					,			_UpperCAmelCase  :       Tuple=None					,			_UpperCAmelCase  :       int=None					,			_UpperCAmelCase  :       Tuple="<s>"					,			_UpperCAmelCase  :       Dict="</s>"					,			_UpperCAmelCase  :       Dict="</s>"					,			_UpperCAmelCase  :       List[Any]="<s>"					,			_UpperCAmelCase  :       List[Any]="<unk>"					,			_UpperCAmelCase  :       Optional[int]="<pad>"					,			_UpperCAmelCase  :       Union[str, Any]="<mask>"					,			**_UpperCAmelCase  :       Optional[int]					,			)						->		Dict:



											"""simple docstring"""


											__lowercase								= AddedToken(_UpperCAmelCase					,			lstrip=_UpperCAmelCase					,			rstrip=_UpperCAmelCase    ) if isinstance(_UpperCAmelCase					,			_UpperCAmelCase    ) else mask_token

											super().__init__(
											    _UpperCAmelCase					,			tokenizer_file=_UpperCAmelCase					,			bos_token=_UpperCAmelCase					,			eos_token=_UpperCAmelCase					,			sep_token=_UpperCAmelCase					,			cls_token=_UpperCAmelCase					,			unk_token=_UpperCAmelCase					,			pad_token=_UpperCAmelCase					,			mask_token=_UpperCAmelCase					,			**_UpperCAmelCase					,			)

											__lowercase								= vocab_file
											__lowercase								= False if not self.vocab_file else True





							def       a__			(			self  :       Union[str, Any]					,			_UpperCAmelCase  :       List[int]					,			_UpperCAmelCase  :       Optional[List[int]] = None    )						->		List[int]:



											"""simple docstring"""


											if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__lowercase								= [self.cls_token_id]
											__lowercase								= [self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep





							def       a__			(			self  :       Union[str, Any]					,			_UpperCAmelCase  :       List[int]					,			_UpperCAmelCase  :       Optional[List[int]] = None    )						->		List[int]:



											"""simple docstring"""


											__lowercase								= [self.sep_token_id]
											__lowercase								= [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]





							def       a__			(			self  :       str					,			_UpperCAmelCase  :       str					,			_UpperCAmelCase  :       Optional[str] = None    )						->		Tuple[str]:



											"""simple docstring"""


											if not self.can_save_slow_tokenizer:
															raise ValueError(
															    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
															    'tokenizer.'    )

											if not os.path.isdir(_UpperCAmelCase    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""    )
															return
											__lowercase								= os.path.join(
											    _UpperCAmelCase					,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(_UpperCAmelCase    ):
															copyfile(self.vocab_file					,			_UpperCAmelCase    )

											return (out_vocab_file,)





