


'''simple docstring'''

import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase							:       List[str]  						=     logging.get_logger(__name__)

lowerCamelCase							:       Tuple  						=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowerCamelCase							:       Optional[int]  						=     {
    "vocab_file": {
        "microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json",
    },
    "merges_file": {"microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"},
}

lowerCamelCase							:       Optional[Any]  						=     {
    "microsoft/biogpt": 1_024,
}

def _SCREAMING_SNAKE_CASE				(A	)	->	str:






       """simple docstring"""







       lowercase__				      =			set()
       lowercase__				      =			word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)	)
              lowercase__				      =			char
       return pairs

class   __lowerCAmelCase						(UpperCAmelCase__					):





      '''simple docstring'''






      lowerCAmelCase__ :					List[str]   =							VOCAB_FILES_NAMES
      lowerCAmelCase__ :					Any   =							PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__ :					Optional[Any]   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__ :					Union[str, Any]   =							["""input_ids""", """attention_mask"""]



      def __init__(self   :   Union[str, Any]       ,						UpperCamelCase   :   Dict       ,						UpperCamelCase   :   Tuple       ,						UpperCamelCase   :   List[Any]="<unk>"       ,						UpperCamelCase   :   Tuple="<s>"       ,						UpperCamelCase   :   Tuple="</s>"       ,						UpperCamelCase   :   Union[str, Any]="</s>"       ,						UpperCamelCase   :   str="<pad>"       ,						**UpperCamelCase   :   List[Any]       ,						):

             '''simple docstring'''







             super().__init__(
                 bos_token=__lowercase       ,						eos_token=__lowercase       ,						sep_token=__lowercase       ,						unk_token=__lowercase       ,						pad_token=__lowercase       ,						**__lowercase       ,						)

             try:
                    import sacremoses
             except ImportError:
                    raise ImportError(
                        '''You need to install sacremoses to use BioGptTokenizer. '''
                        '''See https://pypi.org/project/sacremoses/ for installation.'''			)

             lowercase__				      =			'''en'''
             lowercase__				      =			sacremoses
             # cache of sm.MosesTokenizer instance
             lowercase__				      =			{}
             lowercase__				      =			{}
             with open(__lowercase       ,						encoding='''utf-8'''			) as vocab_handle:
                    lowercase__				      =			json.load(__lowercase			)
             lowercase__				      =			{v: k for k, v in self.encoder.items()}
             with open(__lowercase       ,						encoding='''utf-8'''			) as merges_handle:
                    lowercase__				      =			merges_handle.read().split('''\n'''			)[:-1]
             lowercase__				      =			[tuple(merge.split()[:2]			) for merge in merges]
             lowercase__				      =			dict(zip(__lowercase       ,						range(len(__lowercase			)			)			)			)
             lowercase__				      =			{}



      @property
      def 				UpperCamelCase__		(self   :   List[Any]			):

             '''simple docstring'''







             return len(self.encoder			)



      def 				UpperCamelCase__		(self   :   Optional[int]			):

             '''simple docstring'''







             return dict(self.encoder       ,						**self.added_tokens_encoder			)



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   Dict       ,						UpperCamelCase   :   int			):

             '''simple docstring'''







             if lang not in self.cache_moses_tokenizer:
                    lowercase__				      =			self.sm.MosesTokenizer(lang=__lowercase			)
                    lowercase__				      =			moses_tokenizer
             return self.cache_moses_tokenizer[lang].tokenize(
                 __lowercase       ,						aggressive_dash_splits=__lowercase       ,						return_str=__lowercase       ,						escape=__lowercase			)



      def 				UpperCamelCase__		(self   :   Tuple       ,						UpperCamelCase   :   str       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             if lang not in self.cache_moses_detokenizer:
                    lowercase__				      =			self.sm.MosesDetokenizer(lang=__lowercase			)
                    lowercase__				      =			moses_detokenizer
             return self.cache_moses_detokenizer[lang].detokenize(__lowercase			)



      def 				UpperCamelCase__		(self   :   Tuple       ,						UpperCamelCase   :   Dict			):

             '''simple docstring'''







             lowercase__				      =			tuple(token[:-1]			) + (token[-1] + '''</w>''',)
             if token in self.cache:
                    return self.cache[token]
             lowercase__				      =			get_pairs(__lowercase			)

             if not pairs:
                    return token + "</w>"

             while True:
                    lowercase__				      =			min(__lowercase       ,						key=lambda UpperCamelCase			: self.bpe_ranks.get(__lowercase       ,						float('''inf'''			)			)			)
                    if bigram not in self.bpe_ranks:
                           break
                    lowercase__      ,lowercase__				      =			bigram
                    lowercase__				      =			[]
                    lowercase__				      =			0
                    while i < len(__lowercase			):
                           try:
                                  lowercase__				      =			word.index(__lowercase       ,						__lowercase			)
                           except ValueError:
                                  new_word.extend(word[i:]			)
                                  break
                           else:
                                  new_word.extend(word[i:j]			)
                                  lowercase__				      =			j

                           if word[i] == first and i < len(__lowercase			) - 1 and word[i + 1] == second:
                                  new_word.append(first + second			)
                                  i += 2
                           else:
                                  new_word.append(word[i]			)
                                  i += 1
                    lowercase__				      =			tuple(__lowercase			)
                    lowercase__				      =			new_word
                    if len(__lowercase			) == 1:
                           break
                    else:
                           lowercase__				      =			get_pairs(__lowercase			)
             lowercase__				      =			''' '''.join(__lowercase			)
             if word == "\n  </w>":
                    lowercase__				      =			'''\n</w>'''
             lowercase__				      =			word
             return word



      def 				UpperCamelCase__		(self   :   Optional[Any]       ,						UpperCamelCase   :   int       ,						UpperCamelCase   :   Dict=False			):

             '''simple docstring'''







             if bypass_tokenizer:
                    lowercase__				      =			text.split()
             else:
                    lowercase__				      =			self.moses_tokenize(__lowercase       ,						self.lang			)

             lowercase__				      =			[]
             for token in text:
                    if token:
                           split_tokens.extend(list(self.bpe(__lowercase			).split(''' '''			)			)			)

             return split_tokens



      def 				UpperCamelCase__		(self   :   Optional[int]       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             return self.encoder.get(__lowercase       ,						self.encoder.get(self.unk_token			)			)



      def 				UpperCamelCase__		(self   :   List[Any]       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             return self.decoder.get(__lowercase       ,						self.unk_token			)



      def 				UpperCamelCase__		(self   :   Union[str, Any]       ,						UpperCamelCase   :   str			):

             '''simple docstring'''







             lowercase__				      =			[t.replace(''' '''       ,						''''''			).replace('''</w>'''       ,						''' '''			) for t in tokens]
             lowercase__				      =			''''''.join(__lowercase			).split()
             # detokenize
             lowercase__				      =			self.moses_detokenize(__lowercase       ,						self.lang			)
             return text



      def 				UpperCamelCase__		(self   :   Optional[Any]       ,						UpperCamelCase   :   List[int]       ,						UpperCamelCase   :   Optional[List[int]] = None			):

             '''simple docstring'''







             if token_ids_a is None:
                    return [self.sep_token_id] + token_ids_a
             lowercase__				      =			[self.sep_token_id]
             return sep + token_ids_a + sep + token_ids_a



      def 				UpperCamelCase__		(self   :   str       ,						UpperCamelCase   :   List[int]       ,						UpperCamelCase   :   Optional[List[int]] = None       ,						UpperCamelCase   :   bool = False			):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=__lowercase       ,						token_ids_a=__lowercase       ,						already_has_special_tokens=__lowercase			)
             # no bos used in fairseq
             if token_ids_a is not None:
                    return [1] + ([0] * len(__lowercase			)) + [1] + ([0] * len(__lowercase			))
             return [1] + ([0] * len(__lowercase			))



      def 				UpperCamelCase__		(self   :   Optional[Any]       ,						UpperCamelCase   :   List[int]       ,						UpperCamelCase   :   Optional[List[int]] = None			):

             '''simple docstring'''







             lowercase__				      =			[self.sep_token_id]

             # no bos used in fairseq
             if token_ids_a is None:
                    return len(token_ids_a + sep			) * [0]
             return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   str       ,						UpperCamelCase   :   Optional[str] = None			):

             '''simple docstring'''







             if not os.path.isdir(__lowercase			):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
                    return
             lowercase__				      =			os.path.join(
                 __lowercase       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
             lowercase__				      =			os.path.join(
                 __lowercase       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

             with open(__lowercase       ,						'''w'''       ,						encoding='''utf-8'''			) as f:
                    f.write(json.dumps(self.encoder       ,						indent=2       ,						sort_keys=__lowercase       ,						ensure_ascii=__lowercase			) + '''\n'''			)

             lowercase__				      =			0
             with open(__lowercase       ,						'''w'''       ,						encoding='''utf-8'''			) as writer:
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,						key=lambda UpperCamelCase			: kv[1]			):
                           if index != token_index:
                                  logger.warning(
                                      f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                      ''' Please check that the tokenizer is not corrupted!'''			)
                                  lowercase__				      =			token_index
                           writer.write(''' '''.join(__lowercase			) + '''\n'''			)
                           index += 1

             return vocab_file, merge_file



      def __getstate__(self   :   Tuple			):

             '''simple docstring'''







             lowercase__				      =			self.__dict__.copy()
             lowercase__				      =			None
             return state






      def __setstate__(self   :   List[str]       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             lowercase__				      =			d

             try:
                    import sacremoses
             except ImportError:
                    raise ImportError(
                        '''You need to install sacremoses to use XLMTokenizer. '''
                        '''See https://pypi.org/project/sacremoses/ for installation.'''			)

             lowercase__				      =			sacremoses








'''simple docstring'''

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


lowerCamelCase							:       Any  						=     'Here is a text containing a lot of information: \'\'\'{text}\'\'\'.\n\nCan you answer this question about the text: \'{question}\''
class   __lowerCAmelCase						(lowercase_					):





      '''simple docstring'''






      lowerCAmelCase__ :					Dict   =							"""google/flan-t5-base"""
      lowerCAmelCase__ :					List[Any]   =							(
          """This is a tool that answers questions related to a text. It takes two arguments named `text`, which is the """
          """text where to find the answer, and `question`, which is the question, and returns the answer to the question."""
      )
      lowerCAmelCase__ :					Any   =							"""text_qa"""
      lowerCAmelCase__ :					Union[str, Any]   =							AutoTokenizer
      lowerCAmelCase__ :					Any   =							AutoModelForSeqaSeqLM

      lowerCAmelCase__ :					Any   =							["""text""", """text"""]
      lowerCAmelCase__ :					Optional[int]   =							["""text"""]



      def 				UpperCamelCase__		(self   :   Union[str, Any]       ,						UpperCamelCase   :   str       ,						UpperCamelCase   :   str			):

             '''simple docstring'''







             lowercase__				      =			QA_PROMPT.format(text=UpperCamelCase       ,						question=UpperCamelCase			)
             return self.pre_processor(UpperCamelCase       ,						return_tensors='''pt'''			)



      def 				UpperCamelCase__		(self   :   Dict       ,						UpperCamelCase   :   List[Any]			):

             '''simple docstring'''







             lowercase__				      =			self.model.generate(**UpperCamelCase			)

             lowercase__      ,lowercase__				      =			inputs['''input_ids'''].shape
             lowercase__				      =			output_ids.shape[0]

             return output_ids.reshape(UpperCamelCase       ,						out_b // in_b       ,						*output_ids.shape[1:]			)[0][0]






      def 				UpperCamelCase__		(self   :   Tuple       ,						UpperCamelCase   :   Optional[Any]			):

             '''simple docstring'''







             return self.pre_processor.decode(UpperCamelCase       ,						skip_special_tokens=UpperCamelCase       ,						clean_up_tokenization_spaces=UpperCamelCase			)




