






from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			=      logging.get_logger(__name__)







class 		_a       (						lowercase_   ):





   '''simple docstring'''






   UpperCamelCase__      					=    ["input_ids", "attention_mask"]





   def __init__(   self			,				UpperCAmelCase_="[PAD]"			,				UpperCAmelCase_="[BOS]"			,				UpperCAmelCase_="[EOS]"			,				UpperCAmelCase_="[MASK]"			,				UpperCAmelCase_="[CLS]"			,				UpperCAmelCase_="[SEP]"			,				UpperCAmelCase_=2_048			,				**UpperCAmelCase_			,				)	-> Optional[Any]:




          '''simple docstring'''




          lowercase__:    Any        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else pad_token
          lowercase__:    List[str]        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else bos_token
          lowercase__:    Union[str, Any]        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else eos_token
          lowercase__:    int        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else mask_token
          lowercase__:    Tuple        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else cls_token
          lowercase__:    int        =      AddedToken(__A			,				lstrip=__A			,				rstrip=__A) if isinstance(__A			,				__A) else sep_token

          super().__init__(
              pad_token=__A			,				bos_token=__A			,				eos_token=__A			,				mask_token=__A			,				cls_token=__A			,				sep_token=__A			,				model_max_length=__A			,				**__A			,				)

          lowercase__:    Dict        =      2**8  # utf is 8 bits

          # define special tokens dict
          lowercase__:    Dict[str, int]        =      {
              self.pad_token: 0,
              self.bos_token: 1,
              self.eos_token: 2,
              self.mask_token: 3,
              self.cls_token: 4,
              self.sep_token: 5,
          }
          lowercase__:    Optional[int]        =      len(self.special_tokens_encoder)
          lowercase__:    Dict[int, str]        =      {v: k for k, v in self.special_tokens_encoder.items()}





   def __lowercase    (   self)	-> Union[str, Any]:




          '''simple docstring'''




          lowercase__:    int        =      self.special_tokens_encoder.copy()
          vocab.update(self.added_tokens_encoder)
          for i in range(self._utf_vocab_size):
                 lowercase__:    Dict        =      chr(__A)
                 lowercase__:    int        =      i + len(self.special_tokens_encoder)
          return vocab





   @property
   def __lowercase    (   self)	-> Dict:




          '''simple docstring'''




          return self._utf_vocab_size + self._num_special_tokens





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None			,				UpperCAmelCase_ = False)	-> Optional[Any]:




          '''simple docstring'''




          if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__A			,				token_ids_a=__A			,				already_has_special_tokens=__A)

          # normal case: some special tokens
          if token_ids_a is None:
                 return [1] + [0] * len(__A) + [1]
          return [1] + ([0] * len(__A)) + [1] + ([0] * len(__A)) + [1]





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None)	-> Optional[int]:




          '''simple docstring'''




          if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          else:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Dict:




          '''simple docstring'''




          lowercase__:    List[str]        =      [chr(__A) for i in text.encode("utf-8")]
          return tokens





   def __lowercase    (   self			,				UpperCAmelCase_)	-> str:




          '''simple docstring'''




          if token in self.special_tokens_encoder:
                 lowercase__:    Union[str, Any]        =      self.special_tokens_encoder[token]
          elif token in self.added_tokens_encoder:
                 lowercase__:    Optional[int]        =      self.added_tokens_encoder[token]
          elif len(__A) != 1:
                 lowercase__:    Union[str, Any]        =      self.unk_token_id
          else:
                 lowercase__:    Union[str, Any]        =      ord(__A) + self._num_special_tokens
          return token_id





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Tuple:




          '''simple docstring'''




          if index in self.special_tokens_decoder:
                 lowercase__:    str        =      self.special_tokens_decoder[index]
          elif index in self.added_tokens_decoder:
                 lowercase__:    Dict        =      self.added_tokens_decoder[index]
          else:
                 lowercase__:    List[str]        =      chr(index - self._num_special_tokens)
          return token





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Tuple:




          '''simple docstring'''




          lowercase__:    int        =      B""
          for token in tokens:
                 if token in self.special_tokens_decoder:
                        lowercase__:    int        =      self.special_tokens_decoder[token].encode("utf-8")
                 elif token in self.added_tokens_decoder:
                        lowercase__:    Union[str, Any]        =      self.special_tokens_decoder[token].encode("utf-8")
                 elif token in self.special_tokens_encoder:
                        lowercase__:    str        =      token.encode("utf-8")
                 elif token in self.added_tokens_encoder:
                        lowercase__:    Optional[Any]        =      token.encode("utf-8")
                 else:
                        lowercase__:    List[Any]        =      bytes([ord(__A)])
                 bstring += tok_string
          lowercase__:    int        =      bstring.decode("utf-8"			,				errors="replace")
          return string





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None)	-> Any:




          '''simple docstring'''




          return ()














"""simple docstring"""





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase			=      {
    """agents""": ["""Agent""", """AzureOpenAiAgent""", """HfAgent""", """LocalAgent""", """OpenAiAgent"""],
    """base""": ["""PipelineTool""", """RemoteTool""", """Tool""", """launch_gradio_demo""", """load_tool"""],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase			=      ["""DocumentQuestionAnsweringTool"""]
  UpperCamelCase			=      ["""ImageCaptioningTool"""]
  UpperCamelCase			=      ["""ImageQuestionAnsweringTool"""]
  UpperCamelCase			=      ["""ImageSegmentationTool"""]
  UpperCamelCase			=      ["""SpeechToTextTool"""]
  UpperCamelCase			=      ["""TextClassificationTool"""]
  UpperCamelCase			=      ["""TextQuestionAnsweringTool"""]
  UpperCamelCase			=      ["""TextSummarizationTool"""]
  UpperCamelCase			=      ["""TextToSpeechTool"""]
  UpperCamelCase			=      ["""TranslationTool"""]

if TYPE_CHECKING:
  from .agents import Agent, AzureOpenAiAgent, HfAgent, LocalAgent, OpenAiAgent
  from .base import PipelineTool, RemoteTool, Tool, launch_gradio_demo, load_tool

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .document_question_answering import DocumentQuestionAnsweringTool
    from .image_captioning import ImageCaptioningTool
    from .image_question_answering import ImageQuestionAnsweringTool
    from .image_segmentation import ImageSegmentationTool
    from .speech_to_text import SpeechToTextTool
    from .text_classification import TextClassificationTool
    from .text_question_answering import TextQuestionAnsweringTool
    from .text_summarization import TextSummarizationTool
    from .text_to_speech import TextToSpeechTool
    from .translation import TranslationTool
else:
  import sys

  UpperCamelCase			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






