






"""simple docstring"""





import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			=      logging.get_logger(__name__)

UpperCamelCase			=      {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

UpperCamelCase			=      {
    """vocab_file""": {
        """microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json""",
    },
    """merges_file""": {"""microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"""},
}

UpperCamelCase			=      {
    """microsoft/biogpt""": 10_24,
}






def     A(  snake_case_		):
       """simple docstring"""





       lowercase__:    str        =      set()
       lowercase__:    Optional[int]        =      word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)		)
              lowercase__:    Tuple        =      char
       return pairs







class 		_a       (						lowercase_   ):





   '''simple docstring'''






   UpperCamelCase__      					=    VOCAB_FILES_NAMES
   UpperCamelCase__      					=    PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__      					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase__      					=    ["""input_ids""", """attention_mask"""]





   def __init__(   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_="<unk>"			,				UpperCAmelCase_="<s>"			,				UpperCAmelCase_="</s>"			,				UpperCAmelCase_="</s>"			,				UpperCAmelCase_="<pad>"			,				**UpperCAmelCase_			,				)	-> Any:




          '''simple docstring'''




          super().__init__(
              bos_token=UpperCAmelCase_			,				eos_token=UpperCAmelCase_			,				sep_token=UpperCAmelCase_			,				unk_token=UpperCAmelCase_			,				pad_token=UpperCAmelCase_			,				**UpperCAmelCase_			,				)

          try:
                 import sacremoses
          except ImportError:
                 raise ImportError(
                     "You need to install sacremoses to use BioGptTokenizer. "
                     "See https://pypi.org/project/sacremoses/ for installation.")

          lowercase__:    List[Any]        =      "en"
          lowercase__:    int        =      sacremoses
          # cache of sm.MosesTokenizer instance
          lowercase__:    Optional[Any]        =      {}
          lowercase__:    str        =      {}
          with open(UpperCAmelCase_			,				encoding="utf-8") as vocab_handle:
                 lowercase__:    List[str]        =      json.load(UpperCAmelCase_)
          lowercase__:    Any        =      {v: k for k, v in self.encoder.items()}
          with open(UpperCAmelCase_			,				encoding="utf-8") as merges_handle:
                 lowercase__:    Optional[Any]        =      merges_handle.read().split("\n")[:-1]
          lowercase__:    Optional[int]        =      [tuple(merge.split()[:2]) for merge in merges]
          lowercase__:    Optional[int]        =      dict(zip(UpperCAmelCase_			,				range(len(UpperCAmelCase_))))
          lowercase__:    str        =      {}





   @property
   def __lowercase    (   self)	-> Union[str, Any]:




          '''simple docstring'''




          return len(self.encoder)





   def __lowercase    (   self)	-> Dict:




          '''simple docstring'''




          return dict(self.encoder			,				**self.added_tokens_encoder)





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> Union[str, Any]:




          '''simple docstring'''




          if lang not in self.cache_moses_tokenizer:
                 lowercase__:    List[str]        =      self.sm.MosesTokenizer(lang=UpperCAmelCase_)
                 lowercase__:    List[str]        =      moses_tokenizer
          return self.cache_moses_tokenizer[lang].tokenize(
              UpperCAmelCase_			,				aggressive_dash_splits=UpperCAmelCase_			,				return_str=UpperCAmelCase_			,				escape=UpperCAmelCase_)





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> Any:




          '''simple docstring'''




          if lang not in self.cache_moses_detokenizer:
                 lowercase__:    Optional[Any]        =      self.sm.MosesDetokenizer(lang=UpperCAmelCase_)
                 lowercase__:    Dict        =      moses_detokenizer
          return self.cache_moses_detokenizer[lang].detokenize(UpperCAmelCase_)





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Any:




          '''simple docstring'''




          lowercase__:    Dict        =      tuple(token[:-1]) + (token[-1] + "</w>",)
          if token in self.cache:
                 return self.cache[token]
          lowercase__:    int        =      get_pairs(UpperCAmelCase_)

          if not pairs:
                 return token + "</w>"

          while True:
                 lowercase__:    Any        =      min(UpperCAmelCase_			,				key=lambda UpperCAmelCase_: self.bpe_ranks.get(UpperCAmelCase_			,				float("inf")))
                 if bigram not in self.bpe_ranks:
                        break
                 lowercase__  ,     lowercase__:    Union[str, Any]        =      bigram
                 lowercase__:    Tuple        =      []
                 lowercase__:    List[str]        =      0
                 while i < len(UpperCAmelCase_):
                        try:
                               lowercase__:    Dict        =      word.index(UpperCAmelCase_			,				UpperCAmelCase_)
                        except ValueError:
                               new_word.extend(word[i:])
                               break
                        else:
                               new_word.extend(word[i:j])
                               lowercase__:    Dict        =      j

                        if word[i] == first and i < len(UpperCAmelCase_) - 1 and word[i + 1] == second:
                               new_word.append(first + second)
                               i += 2
                        else:
                               new_word.append(word[i])
                               i += 1
                 lowercase__:    int        =      tuple(UpperCAmelCase_)
                 lowercase__:    List[Any]        =      new_word
                 if len(UpperCAmelCase_) == 1:
                        break
                 else:
                        lowercase__:    Dict        =      get_pairs(UpperCAmelCase_)
          lowercase__:    Optional[int]        =      " ".join(UpperCAmelCase_)
          if word == "\n  </w>":
                 lowercase__:    List[str]        =      "\n</w>"
          lowercase__:    str        =      word
          return word





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_=False)	-> Optional[int]:




          '''simple docstring'''




          if bypass_tokenizer:
                 lowercase__:    int        =      text.split()
          else:
                 lowercase__:    List[Any]        =      self.moses_tokenize(UpperCAmelCase_			,				self.lang)

          lowercase__:    Dict        =      []
          for token in text:
                 if token:
                        split_tokens.extend(list(self.bpe(UpperCAmelCase_).split(" ")))

          return split_tokens





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Optional[int]:




          '''simple docstring'''




          return self.encoder.get(UpperCAmelCase_			,				self.encoder.get(self.unk_token))





   def __lowercase    (   self			,				UpperCAmelCase_)	-> int:




          '''simple docstring'''




          return self.decoder.get(UpperCAmelCase_			,				self.unk_token)





   def __lowercase    (   self			,				UpperCAmelCase_)	-> Optional[Any]:




          '''simple docstring'''




          lowercase__:    List[Any]        =      [t.replace(" "			,				"").replace("</w>"			,				" ") for t in tokens]
          lowercase__:    List[Any]        =      "".join(UpperCAmelCase_).split()
          # detokenize
          lowercase__:    Tuple        =      self.moses_detokenize(UpperCAmelCase_			,				self.lang)
          return text





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None)	-> List[int]:




          '''simple docstring'''




          if token_ids_a is None:
                 return [self.sep_token_id] + token_ids_a
          lowercase__:    int        =      [self.sep_token_id]
          return sep + token_ids_a + sep + token_ids_a





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None			,				UpperCAmelCase_ = False)	-> List[int]:




          '''simple docstring'''




          if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=UpperCAmelCase_			,				token_ids_a=UpperCAmelCase_			,				already_has_special_tokens=UpperCAmelCase_)
          # no bos used in fairseq
          if token_ids_a is not None:
                 return [1] + ([0] * len(UpperCAmelCase_)) + [1] + ([0] * len(UpperCAmelCase_))
          return [1] + ([0] * len(UpperCAmelCase_))





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None)	-> List[int]:




          '''simple docstring'''




          lowercase__:    Union[str, Any]        =      [self.sep_token_id]

          # no bos used in fairseq
          if token_ids_a is None:
                 return len(token_ids_a + sep) * [0]
          return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_ = None)	-> Tuple[str]:




          '''simple docstring'''




          if not os.path.isdir(UpperCAmelCase_):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                 return
          lowercase__:    Optional[Any]        =      os.path.join(
              UpperCAmelCase_			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
          lowercase__:    List[Any]        =      os.path.join(
              UpperCAmelCase_			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

          with open(UpperCAmelCase_			,				"w"			,				encoding="utf-8") as f:
                 f.write(json.dumps(self.encoder			,				indent=2			,				sort_keys=UpperCAmelCase_			,				ensure_ascii=UpperCAmelCase_) + "\n")

          lowercase__:    int        =      0
          with open(UpperCAmelCase_			,				"w"			,				encoding="utf-8") as writer:
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,				key=lambda UpperCAmelCase_: kv[1]):
                        if index != token_index:
                               logger.warning(
                                   F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                   " Please check that the tokenizer is not corrupted!")
                               lowercase__:    List[Any]        =      token_index
                        writer.write(" ".join(UpperCAmelCase_) + "\n")
                        index += 1

          return vocab_file, merge_file





   def __getstate__(   self)	-> Any:




          '''simple docstring'''




          lowercase__:    Dict        =      self.__dict__.copy()
          lowercase__:    Any        =      None
          return state





   def __setstate__(   self			,				UpperCAmelCase_)	-> Optional[Any]:




          '''simple docstring'''




          lowercase__:    str        =      d

          try:
                 import sacremoses
          except ImportError:
                 raise ImportError(
                     "You need to install sacremoses to use XLMTokenizer. "
                     "See https://pypi.org/project/sacremoses/ for installation.")

          lowercase__:    Dict        =      sacremoses














"""simple docstring"""





def     A(  snake_case_						,       snake_case_		):
       """simple docstring"""





       assert isinstance(snake_case_						,       snake_case_		), F"""The input value of [set_a={set_a}] is not a set"""
       assert isinstance(snake_case_						,       snake_case_		), F"""The input value of [set_b={set_b}] is not a set"""

       return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
  from doctest import testmod

  testmod()






