





'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbartaa import MBartaaTokenizer
else:
					snake_case_       :    Tuple          =					None


snake_case_       :    Tuple          =					logging.get_logger(__name__)

snake_case_       :    str          =					{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

snake_case_       :    Any          =					{
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

snake_case_       :    Any          =					{
    "facebook/mbart-large-50-one-to-many-mmt": 10_24,
}

# fmt: off
snake_case_       :    List[str]          =					["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]


class    __a (lowerCamelCase       ):
							__a   :				str       =    VOCAB_FILES_NAMES
							__a   :				Union[str, Any]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a   :				List[str]       =    PRETRAINED_VOCAB_FILES_MAP
							__a   :				Dict       =    ["input_ids", "attention_mask"]
							__a   :				List[Any]       =    MBartaaTokenizer

							__a   :				List[int]       =    []
							__a   :				List[int]       =    []

							def __init__(							self							:       Optional[int]    ,       __magic_name__							:       str=None    ,       __magic_name__							:       Any=None    ,       __magic_name__							:       List[Any]=None    ,       __magic_name__							:       List[Any]=None    ,       __magic_name__							:       Union[str, Any]="</s>"    ,       __magic_name__							:       Optional[Any]="</s>"    ,       __magic_name__							:       List[str]="<s>"    ,       __magic_name__							:       List[Any]="<unk>"    ,       __magic_name__							:       Optional[int]="<pad>"    ,       __magic_name__							:       Dict="<mask>"    ,       **__magic_name__							:       int    ,       )   ->    int:







													"""simple docstring"""
													# Mask token behave like a normal word, i.e. include the space before it
													UpperCAmelCase_     : Union[str, Any]         =					AddedToken(__magic_name__    ,       lstrip=__magic_name__    ,       rstrip=__magic_name__				) if isinstance(__magic_name__    ,       __magic_name__				) else mask_token

													UpperCAmelCase_     : Tuple         =					kwargs.get('''additional_special_tokens'''    ,       []				)
													kwargs["additional_special_tokens"] += [
													    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
													]

													super().__init__(
													    __magic_name__    ,       src_lang=__magic_name__    ,       tgt_lang=__magic_name__    ,       tokenizer_file=__magic_name__    ,       eos_token=__magic_name__    ,       sep_token=__magic_name__    ,       cls_token=__magic_name__    ,       unk_token=__magic_name__    ,       pad_token=__magic_name__    ,       mask_token=__magic_name__    ,       **__magic_name__    ,       )

													UpperCAmelCase_     : Tuple         =					vocab_file
													UpperCAmelCase_     : Dict         =					False if not self.vocab_file else True

													UpperCAmelCase_     : List[str]         =					{
													    lang_code: self.convert_tokens_to_ids(__magic_name__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
													}

													UpperCAmelCase_     : List[str]         =					src_lang if src_lang is not None else '''en_XX'''
													UpperCAmelCase_     : Tuple         =					tgt_lang
													UpperCAmelCase_     : Any         =					self.lang_code_to_id[self._src_lang]
													self.set_src_lang_special_tokens(self._src_lang				)

							@property
							def       UpperCAmelCase__		(							self							:       List[str]				)   ->    str:







													"""simple docstring"""
													return self._src_lang

							@src_lang.setter
							def       UpperCAmelCase__		(							self							:       List[str]    ,       __magic_name__							:       str				)   ->    None:







													"""simple docstring"""
													UpperCAmelCase_     : Tuple         =					new_src_lang
													self.set_src_lang_special_tokens(self._src_lang				)

							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       List[int]    ,       __magic_name__							:       Optional[List[int]] = None				)   ->    List[int]:







													"""simple docstring"""
													if token_ids_a is None:
																			return self.prefix_tokens + token_ids_a + self.suffix_tokens
													# We don't expect to process pairs, but leave the pair logic for API consistency
													return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

							def       UpperCAmelCase__		(							self							:       Tuple    ,       __magic_name__							:       List[str]    ,       __magic_name__							:       str = "en_XX"    ,       __magic_name__							:       Optional[List[str]] = None    ,       __magic_name__							:       str = "ro_RO"    ,       **__magic_name__							:       Union[str, Any]    ,       )   ->    BatchEncoding:







													"""simple docstring"""
													UpperCAmelCase_     : str         =					src_lang
													UpperCAmelCase_     : List[str]         =					tgt_lang
													return super().prepare_seqaseq_batch(__magic_name__    ,       __magic_name__    ,       **__magic_name__				)

							def       UpperCAmelCase__		(							self							:       Optional[int]				)   ->    int:







													"""simple docstring"""
													return self.set_src_lang_special_tokens(self.src_lang				)

							def       UpperCAmelCase__		(							self							:       Any				)   ->    str:







													"""simple docstring"""
													return self.set_tgt_lang_special_tokens(self.tgt_lang				)

							def       UpperCAmelCase__		(							self							:       List[Any]    ,       __magic_name__							:       str				)   ->    None:







													"""simple docstring"""
													UpperCAmelCase_     : Tuple         =					self.convert_tokens_to_ids(__magic_name__				)
													UpperCAmelCase_     : Union[str, Any]         =					[self.cur_lang_code_id]
													UpperCAmelCase_     : List[Any]         =					[self.eos_token_id]

													UpperCAmelCase_     : Optional[Any]         =					self.convert_ids_to_tokens(self.prefix_tokens				)
													UpperCAmelCase_     : int         =					self.convert_ids_to_tokens(self.suffix_tokens				)

													UpperCAmelCase_     : int         =					processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str    ,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str    ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,       self.prefix_tokens + self.suffix_tokens				)				)    ,       )

							def       UpperCAmelCase__		(							self							:       Any    ,       __magic_name__							:       str				)   ->    None:







													"""simple docstring"""
													UpperCAmelCase_     : int         =					self.convert_tokens_to_ids(__magic_name__				)
													UpperCAmelCase_     : Optional[Any]         =					[self.cur_lang_code_id]
													UpperCAmelCase_     : int         =					[self.eos_token_id]

													UpperCAmelCase_     : Any         =					self.convert_ids_to_tokens(self.prefix_tokens				)
													UpperCAmelCase_     : Tuple         =					self.convert_ids_to_tokens(self.suffix_tokens				)

													UpperCAmelCase_     : List[Any]         =					processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str    ,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str    ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,       self.prefix_tokens + self.suffix_tokens				)				)    ,       )

							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       Any    ,       __magic_name__							:       str    ,       __magic_name__							:       Optional[str]    ,       __magic_name__							:       Optional[str]    ,       **__magic_name__							:       Optional[Any]				)   ->    List[str]:







													"""simple docstring"""
													if src_lang is None or tgt_lang is None:
																			raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''				)
													UpperCAmelCase_     : str         =					src_lang
													UpperCAmelCase_     : Union[str, Any]         =					self(__magic_name__    ,       add_special_tokens=__magic_name__    ,       return_tensors=__magic_name__    ,       **__magic_name__				)
													UpperCAmelCase_     : Tuple         =					self.convert_tokens_to_ids(__magic_name__				)
													UpperCAmelCase_     : List[str]         =					tgt_lang_id
													return inputs



							def       UpperCAmelCase__		(							self							:       List[Any]    ,       __magic_name__							:       str    ,       __magic_name__							:       Optional[str] = None				)   ->    Tuple[str]:







													"""simple docstring"""
													if not self.can_save_slow_tokenizer:
																			raise ValueError(
																			    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																			    '''tokenizer.'''				)

													if not os.path.isdir(__magic_name__				):
																			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
																			return
													UpperCAmelCase_     : int         =					os.path.join(
													    __magic_name__    ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

													if os.path.abspath(self.vocab_file				) != os.path.abspath(__magic_name__				):
																			copyfile(self.vocab_file    ,       __magic_name__				)

													return (out_vocab_file,)












'''simple docstring'''







from transformers import SqueezeBertTokenizer, SqueezeBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class    __a (lowerCamelCase       ):
							__a   :				Any       =    SqueezeBertTokenizer
							__a   :				Dict       =    SqueezeBertTokenizerFast
							__a   :				Any       =    True

							def       UpperCAmelCase__		(							self							:       Union[str, Any]    ,       **__magic_name__							:       Dict				)   ->    int:







													"""simple docstring"""
													return SqueezeBertTokenizerFast.from_pretrained(self.tmpdirname    ,       **__magic_name__				)



							@slow
							def       UpperCAmelCase__		(							self							:       Union[str, Any]				)   ->    int:







													"""simple docstring"""
													UpperCAmelCase_     : List[str]         =					SqueezeBertTokenizer.from_pretrained('''squeezebert/squeezebert-mnli-headless'''				)

													UpperCAmelCase_     : Any         =					tokenizer.encode('''sequence builders'''    ,       add_special_tokens=__magic_name__				)
													UpperCAmelCase_     : Optional[Any]         =					tokenizer.encode('''multi-sequence build'''    ,       add_special_tokens=__magic_name__				)

													UpperCAmelCase_     : List[Any]         =					tokenizer.build_inputs_with_special_tokens(__magic_name__				)
													UpperCAmelCase_     : Any         =					tokenizer.build_inputs_with_special_tokens(__magic_name__    ,       __magic_name__				)

													assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
													assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
													    tokenizer.sep_token_id
													]





