





"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__   :       List[str]   						=			logging.get_logger(__name__)


a__   :       Optional[Any]   						=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
a__   :       List[str]   						=			{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

a__   :       List[Any]   						=			{
    """RUCAIBox/mvp""": 1_0_2_4,
}





@lru_cache()
def 							a__     (					)      ->					Tuple:

 '''simple docstring'''






 snake_case__      							=			(
     list(range(ord("!"		)		,       ord("~"		) + 1		)		) + list(range(ord("¡"		)		,       ord("¬"		) + 1		)		) + list(range(ord("®"		)		,       ord("ÿ"		) + 1		)		)
 )
 snake_case__      							=			bs[:]
 snake_case__      							=			0
 for b in range(2**8		):
  if b not in bs:
   bs.append(_UpperCAmelCase		)
   cs.append(2**8 + n		)
   n += 1
 snake_case__      							=			[chr(_UpperCAmelCase		) for n in cs]
 return dict(zip(_UpperCAmelCase		,       _UpperCAmelCase		)		)





def 							a__     (					_UpperCAmelCase		)      ->					int:

 '''simple docstring'''






 snake_case__      							=			set()
 snake_case__      							=			word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)		)
  snake_case__      							=			char
 return pairs



class        lowercase_			(       __lowerCAmelCase							):
       snake_case__			:							str										=       VOCAB_FILES_NAMES
       snake_case__			:							str										=       PRETRAINED_VOCAB_FILES_MAP
       snake_case__			:							int										=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       snake_case__			:							Union[str, Any]										=       ["""input_ids""", """attention_mask"""]

       def __init__(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE="replace"				,   __SCREAMING_SNAKE_CASE="<s>"				,   __SCREAMING_SNAKE_CASE="</s>"				,   __SCREAMING_SNAKE_CASE="</s>"				,   __SCREAMING_SNAKE_CASE="<s>"				,   __SCREAMING_SNAKE_CASE="<unk>"				,   __SCREAMING_SNAKE_CASE="<pad>"				,   __SCREAMING_SNAKE_CASE="<mask>"				,   __SCREAMING_SNAKE_CASE=False				,   **__SCREAMING_SNAKE_CASE				,   ):






        '''simple docstring'''

        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else bos_token
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else eos_token
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else sep_token
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else cls_token
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else unk_token
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        snake_case__      							=			AddedToken(__SCREAMING_SNAKE_CASE				,   lstrip=__SCREAMING_SNAKE_CASE				,   rstrip=__SCREAMING_SNAKE_CASE       ) if isinstance(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       ) else mask_token

        super().__init__(
            errors=__SCREAMING_SNAKE_CASE				,   bos_token=__SCREAMING_SNAKE_CASE				,   eos_token=__SCREAMING_SNAKE_CASE				,   unk_token=__SCREAMING_SNAKE_CASE				,   sep_token=__SCREAMING_SNAKE_CASE				,   cls_token=__SCREAMING_SNAKE_CASE				,   pad_token=__SCREAMING_SNAKE_CASE				,   mask_token=__SCREAMING_SNAKE_CASE				,   add_prefix_space=__SCREAMING_SNAKE_CASE				,   **__SCREAMING_SNAKE_CASE				,   )
        with open(__SCREAMING_SNAKE_CASE				,   encoding="utf-8"       ) as vocab_handle:
         snake_case__      							=			json.load(__SCREAMING_SNAKE_CASE       )
        snake_case__      							=			{v: k for k, v in self.encoder.items()}
        snake_case__      							=			errors  # how to handle errors in decoding
        snake_case__      							=			bytes_to_unicode()
        snake_case__      							=			{v: k for k, v in self.byte_encoder.items()}
        with open(__SCREAMING_SNAKE_CASE				,   encoding="utf-8"       ) as merges_handle:
         snake_case__      							=			merges_handle.read().split("\n"       )[1:-1]
        snake_case__      							=			[tuple(merge.split()       ) for merge in bpe_merges]
        snake_case__      							=			dict(zip(__SCREAMING_SNAKE_CASE				,   range(len(__SCREAMING_SNAKE_CASE       )       )       )       )
        snake_case__      							=			{}
        snake_case__      							=			add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        snake_case__      							=			re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"       )

       @property
       def 						_SCREAMING_SNAKE_CASE(     self       ):






        '''simple docstring'''

        return len(self.encoder       )

       def 						_SCREAMING_SNAKE_CASE(     self       ):






        '''simple docstring'''

        return dict(self.encoder				,   **self.added_tokens_encoder       )

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        if token in self.cache:
         return self.cache[token]
        snake_case__      							=			tuple(__SCREAMING_SNAKE_CASE       )
        snake_case__      							=			get_pairs(__SCREAMING_SNAKE_CASE       )

        if not pairs:
         return token

        while True:
         snake_case__      							=			min(__SCREAMING_SNAKE_CASE				,   key=lambda __SCREAMING_SNAKE_CASE       : self.bpe_ranks.get(__SCREAMING_SNAKE_CASE				,   float("inf"       )       )       )
         if bigram not in self.bpe_ranks:
          break
         snake_case__							,				snake_case__      							=			bigram
         snake_case__      							=			[]
         snake_case__      							=			0
         while i < len(__SCREAMING_SNAKE_CASE       ):
          try:
           snake_case__      							=			word.index(__SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE       )
          except ValueError:
           new_word.extend(word[i:]       )
           break
          else:
           new_word.extend(word[i:j]       )
           snake_case__      							=			j

          if word[i] == first and i < len(__SCREAMING_SNAKE_CASE       ) - 1 and word[i + 1] == second:
           new_word.append(first + second       )
           i += 2
          else:
           new_word.append(word[i]       )
           i += 1
         snake_case__      							=			tuple(__SCREAMING_SNAKE_CASE       )
         snake_case__      							=			new_word
         if len(__SCREAMING_SNAKE_CASE       ) == 1:
          break
         else:
          snake_case__      							=			get_pairs(__SCREAMING_SNAKE_CASE       )
        snake_case__      							=			" ".join(__SCREAMING_SNAKE_CASE       )
        snake_case__      							=			word
        return word

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        snake_case__      							=			[]
        for token in re.findall(self.pat				,   __SCREAMING_SNAKE_CASE       ):
         snake_case__      							=			"".join(
             self.byte_encoder[b] for b in token.encode("utf-8"       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__SCREAMING_SNAKE_CASE       ).split(" "       )       )
        return bpe_tokens

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        return self.encoder.get(__SCREAMING_SNAKE_CASE				,   self.encoder.get(self.unk_token       )       )

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        return self.decoder.get(__SCREAMING_SNAKE_CASE       )

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        snake_case__      							=			"".join(__SCREAMING_SNAKE_CASE       )
        snake_case__      							=			bytearray([self.byte_decoder[c] for c in text]       ).decode("utf-8"				,   errors=self.errors       )
        return text

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE = None       ):






        '''simple docstring'''

        if not os.path.isdir(__SCREAMING_SNAKE_CASE       ):
         logger.error(F"Vocabulary path ({save_directory}) should be a directory"       )
         return
        snake_case__      							=			os.path.join(
            __SCREAMING_SNAKE_CASE				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
        snake_case__      							=			os.path.join(
            __SCREAMING_SNAKE_CASE				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

        with open(__SCREAMING_SNAKE_CASE				,   "w"				,   encoding="utf-8"       ) as f:
         f.write(json.dumps(self.encoder				,   indent=2				,   sort_keys=__SCREAMING_SNAKE_CASE				,   ensure_ascii=__SCREAMING_SNAKE_CASE       ) + "\n"       )

        snake_case__      							=			0
        with open(__SCREAMING_SNAKE_CASE				,   "w"				,   encoding="utf-8"       ) as writer:
         writer.write("#version: 0.2\n"       )
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,   key=lambda __SCREAMING_SNAKE_CASE       : kv[1]       ):
          if index != token_index:
           logger.warning(
               F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
               " Please check that the tokenizer is not corrupted!"       )
           snake_case__      							=			token_index
          writer.write(" ".join(__SCREAMING_SNAKE_CASE       ) + "\n"       )
          index += 1

        return vocab_file, merge_file

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE = None       ):






        '''simple docstring'''

        if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        snake_case__      							=			[self.cls_token_id]
        snake_case__      							=			[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE = None				,   __SCREAMING_SNAKE_CASE = False       ):






        '''simple docstring'''

        if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=__SCREAMING_SNAKE_CASE				,   token_ids_a=__SCREAMING_SNAKE_CASE				,   already_has_special_tokens=__SCREAMING_SNAKE_CASE       )

        if token_ids_a is None:
         return [1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1]
        return [1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1]

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE = None       ):






        '''simple docstring'''

        snake_case__      							=			[self.sep_token_id]
        snake_case__      							=			[self.cls_token_id]

        if token_ids_a is None:
         return len(cls + token_ids_a + sep       ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]

       def 						_SCREAMING_SNAKE_CASE(     self				,   __SCREAMING_SNAKE_CASE				,   __SCREAMING_SNAKE_CASE=False				,   **__SCREAMING_SNAKE_CASE       ):






        '''simple docstring'''

        snake_case__      							=			kwargs.pop("add_prefix_space"				,   self.add_prefix_space       )
        if (is_split_into_words or add_prefix_space) and (len(__SCREAMING_SNAKE_CASE       ) > 0 and not text[0].isspace()):
         snake_case__      							=			" " + text
        return (text, kwargs)








"""simple docstring"""


import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__   :       Optional[Any]   						=			logging.get_logger(__name__)

a__   :       List[str]   						=			{}



class        lowercase_			(       __lowerCAmelCase							):
       snake_case__			:							List[str]										=       """fsmt_decoder"""

       def __init__(     self				,   __SCREAMING_SNAKE_CASE=0				,   __SCREAMING_SNAKE_CASE=0       ):






        '''simple docstring'''

        super().__init__()
        snake_case__      							=			vocab_size
        snake_case__      							=			bos_token_id



class        lowercase_			(       __lowerCAmelCase							):
       snake_case__			:							str										=       """fsmt"""
       snake_case__			:							List[Any]										=       {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}

       def __init__(     self				,   __SCREAMING_SNAKE_CASE=["en", "de"]				,   __SCREAMING_SNAKE_CASE=4_2024				,   __SCREAMING_SNAKE_CASE=4_2024				,   __SCREAMING_SNAKE_CASE="relu"				,   __SCREAMING_SNAKE_CASE=1024				,   __SCREAMING_SNAKE_CASE=200				,   __SCREAMING_SNAKE_CASE=1024				,   __SCREAMING_SNAKE_CASE=4096				,   __SCREAMING_SNAKE_CASE=12				,   __SCREAMING_SNAKE_CASE=16				,   __SCREAMING_SNAKE_CASE=0.0				,   __SCREAMING_SNAKE_CASE=4096				,   __SCREAMING_SNAKE_CASE=12				,   __SCREAMING_SNAKE_CASE=16				,   __SCREAMING_SNAKE_CASE=0.0				,   __SCREAMING_SNAKE_CASE=0.0				,   __SCREAMING_SNAKE_CASE=0.1				,   __SCREAMING_SNAKE_CASE=0.0				,   __SCREAMING_SNAKE_CASE=0.02				,   __SCREAMING_SNAKE_CASE=2				,   __SCREAMING_SNAKE_CASE=True				,   __SCREAMING_SNAKE_CASE=True				,   __SCREAMING_SNAKE_CASE=False				,   __SCREAMING_SNAKE_CASE=5				,   __SCREAMING_SNAKE_CASE=1.0				,   __SCREAMING_SNAKE_CASE=False				,   __SCREAMING_SNAKE_CASE=True				,   __SCREAMING_SNAKE_CASE=1				,   __SCREAMING_SNAKE_CASE=0				,   __SCREAMING_SNAKE_CASE=2				,   __SCREAMING_SNAKE_CASE=2				,   **__SCREAMING_SNAKE_CASE				,   ):






        '''simple docstring'''

        snake_case__      							=			langs
        snake_case__      							=			src_vocab_size
        snake_case__      							=			tgt_vocab_size
        snake_case__      							=			d_model  # encoder_embed_dim and decoder_embed_dim

        snake_case__      							=			encoder_ffn_dim
        snake_case__      							=			snake_case__      							=			encoder_layers
        snake_case__      							=			encoder_attention_heads
        snake_case__      							=			encoder_layerdrop
        snake_case__      							=			decoder_layerdrop
        snake_case__      							=			decoder_ffn_dim
        snake_case__      							=			decoder_layers
        snake_case__      							=			decoder_attention_heads
        snake_case__      							=			max_position_embeddings
        snake_case__      							=			init_std  # Normal(0, this parameter)
        snake_case__      							=			activation_function

        snake_case__      							=			DecoderConfig(vocab_size=__SCREAMING_SNAKE_CASE				,   bos_token_id=__SCREAMING_SNAKE_CASE       )
        if "decoder" in common_kwargs:
         del common_kwargs["decoder"]

        snake_case__      							=			scale_embedding  # scale factor will be sqrt(d_model) if True

        # 3 Types of Dropout
        snake_case__      							=			attention_dropout
        snake_case__      							=			activation_dropout
        snake_case__      							=			dropout

        snake_case__      							=			use_cache
        super().__init__(
            pad_token_id=__SCREAMING_SNAKE_CASE				,   bos_token_id=__SCREAMING_SNAKE_CASE				,   eos_token_id=__SCREAMING_SNAKE_CASE				,   decoder_start_token_id=__SCREAMING_SNAKE_CASE				,   is_encoder_decoder=__SCREAMING_SNAKE_CASE				,   tie_word_embeddings=__SCREAMING_SNAKE_CASE				,   forced_eos_token_id=__SCREAMING_SNAKE_CASE				,   max_length=__SCREAMING_SNAKE_CASE				,   num_beams=__SCREAMING_SNAKE_CASE				,   length_penalty=__SCREAMING_SNAKE_CASE				,   early_stopping=__SCREAMING_SNAKE_CASE				,   **__SCREAMING_SNAKE_CASE				,   )

       def 						_SCREAMING_SNAKE_CASE(     self       ):






        '''simple docstring'''

        snake_case__      							=			copy.deepcopy(self.__dict__       )
        snake_case__      							=			self.decoder.to_dict()
        snake_case__      							=			self.__class__.model_type
        return output

