


'''simple docstring'''







import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


A__:   Dict    =	logging.getLogger(__name__)







class A__	:






      def __init__(			self      :Optional[Any]     )	-> Optional[Any]:




             '''simple docstring'''
             _a   :			int					 =False






      def __UpperCAmelCase	(			self      :List[str]					,						SCREAMING_SNAKE_CASE      :Any					,						SCREAMING_SNAKE_CASE      :Optional[Any]					,						SCREAMING_SNAKE_CASE      :Union[str, Any]					,						SCREAMING_SNAKE_CASE      :Optional[Any]     )	-> Optional[Any]:




             '''simple docstring'''
             if not self.initialized:
                    _a   :			Dict					 =RagRetriever(
                        SCREAMING_SNAKE_CASE					,						question_encoder_tokenizer=SCREAMING_SNAKE_CASE					,						generator_tokenizer=SCREAMING_SNAKE_CASE					,						index=SCREAMING_SNAKE_CASE					,						init_retrieval=SCREAMING_SNAKE_CASE					,						)
                    _a   :			str					 =True






      def __UpperCAmelCase	(			self      :str     )	-> List[str]:




             '''simple docstring'''
             self.retriever.index.init_index()






      def __UpperCAmelCase	(			self      :Tuple     )	-> List[str]:




             '''simple docstring'''
             # delete the old self.retriever object before assigning the new index
             del self.retriever
             _a   :			Optional[Any]					 =False






      def __UpperCAmelCase	(			self      :Any					,						SCREAMING_SNAKE_CASE      :str					,						SCREAMING_SNAKE_CASE      :Dict     )	-> List[str]:




             '''simple docstring'''
             _a ,   _a   :			str					 =self.retriever._main_retrieve(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE     )
             _a   :			List[Any]					 =self.retriever.index.get_doc_dicts(SCREAMING_SNAKE_CASE     )
             return doc_ids, retrieved_doc_embeds, doc_dicts







class A__	(						UpperCAmelCase__				):






      def __init__(			self      :Dict					,						SCREAMING_SNAKE_CASE      :str					,						SCREAMING_SNAKE_CASE      :Optional[int]					,						SCREAMING_SNAKE_CASE      :Optional[Any]					,						SCREAMING_SNAKE_CASE      :Union[str, Any]					,						SCREAMING_SNAKE_CASE      :str=None     )	-> List[str]:




             '''simple docstring'''
             if index is not None and index.is_initialized() and len(SCREAMING_SNAKE_CASE     ) > 0:
                    raise ValueError(
                        """When using Ray for distributed fine-tuning, """
                        """you'll need to provide the paths instead, """
                        """as the dataset and the index are loaded """
                        """separately. More info in examples/rag/use_own_knowledge_dataset.py """     )

             super().__init__(
                 SCREAMING_SNAKE_CASE					,						question_encoder_tokenizer=SCREAMING_SNAKE_CASE					,						generator_tokenizer=SCREAMING_SNAKE_CASE					,						index=SCREAMING_SNAKE_CASE					,						init_retrieval=SCREAMING_SNAKE_CASE					,						)

             _a   :			str					 =retrieval_workers
             _a   :			Dict					 =question_encoder_tokenizer
             _a   :			Union[str, Any]					 =generator_tokenizer
             if len(self.retrieval_workers     ) > 0:
                    ray.get(
                        [
                            worker.create_rag_retriever.remote(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE     )
                            for worker in self.retrieval_workers
                        ]     )






      def __UpperCAmelCase	(			self      :Union[str, Any]     )	-> Dict:




             '''simple docstring'''
             logger.info("""initializing retrieval"""     )

             if len(self.retrieval_workers     ) > 0:
                    ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]     )
             else:
                    # Non-distributed training. Load index into this same process.
                    self.index.init_index()






      def __UpperCAmelCase	(			self      :Tuple					,						SCREAMING_SNAKE_CASE      :str					,						SCREAMING_SNAKE_CASE      :Dict     )	-> Union[str, Any]:




             '''simple docstring'''
             if len(self.retrieval_workers     ) > 0:
                    # Select a random retrieval actor.
                    _a   :			int					 =self.retrieval_workers[random.randint(0					,						len(self.retrieval_workers     ) - 1     )]
                    _a ,   _a ,   _a   :			List[Any]					 =ray.get(
                        random_worker.retrieve.remote(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE     )     )
             else:
                    _a ,   _a   :			Any					 =self._main_retrieve(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE     )
                    _a   :			int					 =self.index.get_doc_dicts(SCREAMING_SNAKE_CASE     )
             return retrieved_doc_embeds, doc_ids, doc_dicts






      @classmethod
      def __UpperCAmelCase	(			cls      :Dict					,						SCREAMING_SNAKE_CASE      :List[str]					,						SCREAMING_SNAKE_CASE      :List[Any]=None					,						**SCREAMING_SNAKE_CASE      :int     )	-> Union[str, Any]:




             '''simple docstring'''
             return super(SCREAMING_SNAKE_CASE					,						cls     ).get_tokenizers(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						**SCREAMING_SNAKE_CASE     )






      @classmethod
      def __UpperCAmelCase	(			cls      :Optional[int]					,						SCREAMING_SNAKE_CASE      :Tuple					,						SCREAMING_SNAKE_CASE      :Any					,						SCREAMING_SNAKE_CASE      :List[str]=None					,						**SCREAMING_SNAKE_CASE      :int     )	-> Union[str, Any]:




             '''simple docstring'''
             _a   :			List[Any]					 =kwargs.pop("""config"""					,						SCREAMING_SNAKE_CASE     ) or RagConfig.from_pretrained(SCREAMING_SNAKE_CASE					,						**SCREAMING_SNAKE_CASE     )
             _a   :			List[Any]					 =RagTokenizer.from_pretrained(SCREAMING_SNAKE_CASE					,						config=SCREAMING_SNAKE_CASE     )
             _a   :			Optional[int]					 =rag_tokenizer.question_encoder
             _a   :			Tuple					 =rag_tokenizer.generator

             if indexed_dataset is not None:
                    _a   :			List[Any]					 ="""custom"""
                    _a   :			int					 =CustomHFIndex(config.retrieval_vector_size					,						SCREAMING_SNAKE_CASE     )
             else:
                    _a   :			Any					 =cls._build_index(SCREAMING_SNAKE_CASE     )

             return cls(
                 SCREAMING_SNAKE_CASE					,						question_encoder_tokenizer=SCREAMING_SNAKE_CASE					,						generator_tokenizer=SCREAMING_SNAKE_CASE					,						retrieval_workers=SCREAMING_SNAKE_CASE					,						index=SCREAMING_SNAKE_CASE					,						)






      def __UpperCAmelCase	(			self      :str     )	-> Optional[Any]:




             '''simple docstring'''
             logger.info("""re-loading the new dataset with embeddings"""     )
             # access from the training loop

             ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]     )

             # build the index object again
             _a   :			int					 =self._build_index(self.config     )

             ray.get(
                 [
                     worker.create_rag_retriever.remote(
                         self.config					,						self.question_encoder_tokenizer					,						self.generator_tokenizer					,						SCREAMING_SNAKE_CASE     )
                     for worker in self.retrieval_workers
                 ]     )






'''simple docstring'''







import argparse
import logging
import sys
import time

import tensorflow as tf
from datasets import load_dataset

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification


if __name__ == "__main__":
   A__:   Optional[Any]    =	argparse.ArgumentParser()

   # Hyperparameters sent by the client are passed as command-line arguments to the script.
   parser.add_argument('''--epochs''', type=int, default=1)
   parser.add_argument('''--per_device_train_batch_size''', type=int, default=16)
   parser.add_argument('''--per_device_eval_batch_size''', type=int, default=8)
   parser.add_argument('''--model_name_or_path''', type=str)
   parser.add_argument('''--learning_rate''', type=str, default=5E-5)
   parser.add_argument('''--do_train''', type=bool, default=True)
   parser.add_argument('''--do_eval''', type=bool, default=True)
   parser.add_argument('''--output_dir''', type=str)

   A__   ,	A__:   List[Any]    =	parser.parse_known_args()

   # overwrite batch size until we have tf_glue.py
   A__:   Tuple    =	16
   A__:   List[str]    =	16

   # Set up logging
   A__:   str    =	logging.getLogger(__name__)

   logging.basicConfig(
       level=logging.getLevelName('''INFO'''),
       handlers=[logging.StreamHandler(sys.stdout)],
       format='''%(asctime)s - %(name)s - %(levelname)s - %(message)s''',
   )

   # Load model and tokenizer
   A__:   int    =	TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
   A__:   Any    =	AutoTokenizer.from_pretrained(args.model_name_or_path)

   # Load dataset
   A__   ,	A__:   Optional[int]    =	load_dataset('''imdb''', split=['''train''', '''test'''])
   A__:   int    =	train_dataset.shuffle().select(range(5000))  # smaller the size for train dataset to 5k
   A__:   int    =	test_dataset.shuffle().select(range(500))  # smaller the size for test dataset to 500

   # Preprocess train dataset
   A__:   Optional[int]    =	train_dataset.map(
       lambda e: tokenizer(e['''text'''], truncation=True, padding='''max_length'''), batched=True
   )
   train_dataset.set_format(type='''tensorflow''', columns=['''input_ids''', '''attention_mask''', '''label'''])

   A__:   List[Any]    =	{
       x: train_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
       for x in ['''input_ids''', '''attention_mask''']
   }
   A__:   str    =	tf.data.Dataset.from_tensor_slices((train_features, train_dataset['''label'''])).batch(
       args.per_device_train_batch_size
   )

   # Preprocess test dataset
   A__:   str    =	test_dataset.map(
       lambda e: tokenizer(e['''text'''], truncation=True, padding='''max_length'''), batched=True
   )
   test_dataset.set_format(type='''tensorflow''', columns=['''input_ids''', '''attention_mask''', '''label'''])

   A__:   Tuple    =	{
       x: test_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
       for x in ['''input_ids''', '''attention_mask''']
   }
   A__:   Any    =	tf.data.Dataset.from_tensor_slices((test_features, test_dataset['''label'''])).batch(
       args.per_device_eval_batch_size
   )

   # fine optimizer and loss
   A__:   Optional[int]    =	tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
   A__:   Union[str, Any]    =	tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
   A__:   List[Any]    =	[tf.keras.metrics.SparseCategoricalAccuracy()]
   model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

   A__:   str    =	time.time()
   A__:   Dict    =	model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.per_device_train_batch_size)
   A__:   Tuple    =	time.time() - start_train_time

   logger.info('''*** Train ***''')
   logger.info(F"train_runtime = {end_train_time}")
   for key, value in train_results.history.items():
      logger.info(F"  {key} = {value}")


