





'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__							:							int			=logging.get_logger(__name__)

a__							:							List[Any]			={
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}

class 					snake_case					(   A__	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : str      		="opt"
 SCREAMING_SNAKE_CASE_     : int      		=["past_key_values"]






 def __init__(	self	:							List[Any]							,     __A	:							Union[str, Any]=5_0_2_7_2							,     __A	:							Optional[Any]=7_6_8							,     __A	:							Union[str, Any]=1_2							,     __A	:							Any=3_0_7_2							,     __A	:							Tuple=2_0_4_8							,     __A	:							str=True							,     __A	:							Union[str, Any]=False							,     __A	:							Any=None							,     __A	:							List[str]=0.1							,     __A	:							int=0.0							,     __A	:							Any=1_2							,     __A	:							List[str]="relu"							,     __A	:							List[Any]=0.0							,     __A	:							int=0.02							,     __A	:							Any=True							,     __A	:							List[Any]=1							,     __A	:							str=2							,     __A	:							Tuple=2							,     __A	:							Tuple=True							,     __A	:							str=True							,     **__A	:							Any							,     ):
        super().__init__(
            pad_token_id=snake_case_							,     bos_token_id=snake_case_							,     eos_token_id=snake_case_							,     **snake_case_							,     )
        __UpperCamelCase	=  vocab_size
        __UpperCamelCase	=  max_position_embeddings
        __UpperCamelCase	=  num_attention_heads
        __UpperCamelCase	=  word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
        __UpperCamelCase	=  ffn_dim
        __UpperCamelCase	=  hidden_size
        __UpperCamelCase	=  num_hidden_layers
        __UpperCamelCase	=  dropout
        __UpperCamelCase	=  attention_dropout
        __UpperCamelCase	=  activation_function
        __UpperCamelCase	=  init_std
        __UpperCamelCase	=  layerdrop
        __UpperCamelCase	=  use_cache
        __UpperCamelCase	=  do_layer_norm_before
        # We keep these variables at `True` for backward compatibility.
        __UpperCamelCase	=  enable_bias
        __UpperCamelCase	=  layer_norm_elementwise_affine

        # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
        # with checkpoints that have been fine-tuned before transformers v4.20.1
        # see https://github.com/facebookresearch/metaseq/pull/164
        __UpperCamelCase	=  _remove_final_layer_norm







'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbartaa import MBartaaTokenizer
else:
    a__							:							Union[str, Any]			=None


a__							:							str			=logging.get_logger(__name__)

a__							:							Optional[Any]			={'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

a__							:							List[Any]			={
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

a__							:							List[Any]			={
    '''facebook/mbart-large-50-one-to-many-mmt''': 1_024,
}

# fmt: off
a__							:							int			=['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']

class 					snake_case					(   __lowerCamelCase	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : Any      		=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_     : Dict      		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_     : List[str]      		=PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_     : Any      		=["input_ids", "attention_mask"]
 SCREAMING_SNAKE_CASE_     : Dict      		=MBartaaTokenizer

 SCREAMING_SNAKE_CASE_     : List[int]      		=[]
 SCREAMING_SNAKE_CASE_     : List[int]      		=[]






 def __init__(	self	:							Any							,     __A	:							Union[str, Any]=None							,     __A	:							Tuple=None							,     __A	:							Optional[int]=None							,     __A	:							List[Any]=None							,     __A	:							Optional[Any]="</s>"							,     __A	:							Any="</s>"							,     __A	:							Optional[Any]="<s>"							,     __A	:							List[Any]="<unk>"							,     __A	:							Tuple="<pad>"							,     __A	:							Union[str, Any]="<mask>"							,     **__A	:							Any							,     ):
        # Mask token behave like a normal word, i.e. include the space before it
        __UpperCamelCase	=  AddedToken(__A							,     lstrip=__A							,     rstrip=__A     ) if isinstance(__A							,     __A     ) else mask_token

        __UpperCamelCase	=  kwargs.get('additional_special_tokens'							,     []     )
        kwargs["additional_special_tokens"] += [
            code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            __A							,     src_lang=__A							,     tgt_lang=__A							,     tokenizer_file=__A							,     eos_token=__A							,     sep_token=__A							,     cls_token=__A							,     unk_token=__A							,     pad_token=__A							,     mask_token=__A							,     **__A							,     )

        __UpperCamelCase	=  vocab_file
        __UpperCamelCase	=  False if not self.vocab_file else True

        __UpperCamelCase	=  {
            lang_code: self.convert_tokens_to_ids(__A     ) for lang_code in FAIRSEQ_LANGUAGE_CODES
        }

        __UpperCamelCase	=  src_lang if src_lang is not None else 'en_XX'
        __UpperCamelCase	=  tgt_lang
        __UpperCamelCase	=  self.lang_code_to_id[self._src_lang]
        self.set_src_lang_special_tokens(self._src_lang     )






 @property
 def 	_lowerCamelCase (	self	:							Tuple     ):
        return self._src_lang






 @src_lang.setter
 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							str     ):
        __UpperCamelCase	=  new_src_lang
        self.set_src_lang_special_tokens(self._src_lang     )






 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							List[int]							,     __A	:							Optional[List[int]] = None     ):
        if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






 def 	_lowerCamelCase (	self	:							List[Any]							,     __A	:							List[str]							,     __A	:							str = "en_XX"							,     __A	:							Optional[List[str]] = None							,     __A	:							str = "ro_RO"							,     **__A	:							Optional[int]							,     ):
        __UpperCamelCase	=  src_lang
        __UpperCamelCase	=  tgt_lang
        return super().prepare_seqaseq_batch(__A							,     __A							,     **__A     )






 def 	_lowerCamelCase (	self	:							List[str]     ):
        return self.set_src_lang_special_tokens(self.src_lang     )






 def 	_lowerCamelCase (	self	:							List[str]     ):
        return self.set_tgt_lang_special_tokens(self.tgt_lang     )






 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							str     ):
        __UpperCamelCase	=  self.convert_tokens_to_ids(__A     )
        __UpperCamelCase	=  [self.cur_lang_code_id]
        __UpperCamelCase	=  [self.eos_token_id]

        __UpperCamelCase	=  self.convert_ids_to_tokens(self.prefix_tokens     )
        __UpperCamelCase	=  self.convert_ids_to_tokens(self.suffix_tokens     )

        __UpperCamelCase	=  processors.TemplateProcessing(
            single=prefix_tokens_str + ['$A'] + suffix_tokens_str							,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str							,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,     self.prefix_tokens + self.suffix_tokens     )     )							,     )






 def 	_lowerCamelCase (	self	:							Union[str, Any]							,     __A	:							str     ):
        __UpperCamelCase	=  self.convert_tokens_to_ids(__A     )
        __UpperCamelCase	=  [self.cur_lang_code_id]
        __UpperCamelCase	=  [self.eos_token_id]

        __UpperCamelCase	=  self.convert_ids_to_tokens(self.prefix_tokens     )
        __UpperCamelCase	=  self.convert_ids_to_tokens(self.suffix_tokens     )

        __UpperCamelCase	=  processors.TemplateProcessing(
            single=prefix_tokens_str + ['$A'] + suffix_tokens_str							,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str							,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,     self.prefix_tokens + self.suffix_tokens     )     )							,     )






 def 	_lowerCamelCase (	self	:							str							,     __A	:							Union[str, Any]							,     __A	:							str							,     __A	:							Optional[str]							,     __A	:							Optional[str]							,     **__A	:							Dict     ):
        if src_lang is None or tgt_lang is None:
               raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'     )
        __UpperCamelCase	=  src_lang
        __UpperCamelCase	=  self(__A							,     add_special_tokens=__A							,     return_tensors=__A							,     **__A     )
        __UpperCamelCase	=  self.convert_tokens_to_ids(__A     )
        __UpperCamelCase	=  tgt_lang_id
        return inputs






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							str							,     __A	:							Optional[str] = None     ):
        if not self.can_save_slow_tokenizer:
               raise ValueError(
                   'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                   'tokenizer.'     )

        if not os.path.isdir(__A     ):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
               return
        __UpperCamelCase	=  os.path.join(
            __A							,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

        if os.path.abspath(self.vocab_file     ) != os.path.abspath(__A     ):
               copyfile(self.vocab_file							,     __A     )

        return (out_vocab_file,)
