


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbartaa import MBartaaTokenizer
else:
    A				:   Any		= None


A				:   List[Any]		= logging.get_logger(__name__)

A				:   Optional[Any]		= {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

A				:   Any		= {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

A				:   Any		= {
    "facebook/mbart-large-50-one-to-many-mmt": 1024,
}

# fmt: off
A				:   Optional[Any]		= ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]







class       _lowercase			(							lowercase__):


       """simple docstring"""




       A__ 						=      VOCAB_FILES_NAMES
       A__ 						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A__ 						=      PRETRAINED_VOCAB_FILES_MAP
       A__ 						=      ["input_ids", "attention_mask"]
       A__ 						=      MBartaaTokenizer

       A__ 						=      []
       A__ 						=      []





       def __init__(       self							:      Optional[int] ,				__lowerCamelCase							:      List[str]=None ,				__lowerCamelCase							:      Optional[Any]=None ,				__lowerCamelCase							:      Optional[int]=None ,				__lowerCamelCase							:      Dict=None ,				__lowerCamelCase							:      int="</s>" ,				__lowerCamelCase							:      Optional[int]="</s>" ,				__lowerCamelCase							:      Optional[int]="<s>" ,				__lowerCamelCase							:      List[str]="<unk>" ,				__lowerCamelCase							:      List[str]="<pad>" ,				__lowerCamelCase							:      Any="<mask>" ,				**__lowerCamelCase							:      Optional[int] ,				):


             '''simple docstring'''



             lowerCamelCase__		:    Optional[Any]   =  AddedToken(__lowerCamelCase ,				lstrip=__lowerCamelCase ,				rstrip=__lowerCamelCase	) if isinstance(__lowerCamelCase ,				__lowerCamelCase	) else mask_token

             lowerCamelCase__		:    Optional[Any]   =  kwargs.get("additional_special_tokens" ,				[]	)
             kwargs["additional_special_tokens"] += [
                 code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
             ]

             super().__init__(
                 __lowerCamelCase ,				src_lang=__lowerCamelCase ,				tgt_lang=__lowerCamelCase ,				tokenizer_file=__lowerCamelCase ,				eos_token=__lowerCamelCase ,				sep_token=__lowerCamelCase ,				cls_token=__lowerCamelCase ,				unk_token=__lowerCamelCase ,				pad_token=__lowerCamelCase ,				mask_token=__lowerCamelCase ,				**__lowerCamelCase ,				)

             lowerCamelCase__		:    Any   =  vocab_file
             lowerCamelCase__		:    Optional[Any]   =  False if not self.vocab_file else True

             lowerCamelCase__		:    List[Any]   =  {
                 lang_code: self.convert_tokens_to_ids(__lowerCamelCase	) for lang_code in FAIRSEQ_LANGUAGE_CODES
             }

             lowerCamelCase__		:    str   =  src_lang if src_lang is not None else "en_XX"
             lowerCamelCase__		:    List[str]   =  tgt_lang
             lowerCamelCase__		:    Union[str, Any]   =  self.lang_code_to_id[self._src_lang]
             self.set_src_lang_special_tokens(self._src_lang	)





       @property
       def 						lowerCAmelCase					(       self							:      str	):


             '''simple docstring'''



             return self._src_lang





       @src_lang.setter
       def 						lowerCAmelCase					(       self							:      List[Any] ,				__lowerCamelCase							:      str	):


             '''simple docstring'''



             lowerCamelCase__		:    List[Any]   =  new_src_lang
             self.set_src_lang_special_tokens(self._src_lang	)





       def 						lowerCAmelCase					(       self							:      str ,				__lowerCamelCase							:      List[int] ,				__lowerCamelCase							:      Optional[List[int]] = None	):


             '''simple docstring'''



             if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + self.suffix_tokens
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





       def 						lowerCAmelCase					(       self							:      Union[str, Any] ,				__lowerCamelCase							:      List[str] ,				__lowerCamelCase							:      str = "en_XX" ,				__lowerCamelCase							:      Optional[List[str]] = None ,				__lowerCamelCase							:      str = "ro_RO" ,				**__lowerCamelCase							:      Tuple ,				):


             '''simple docstring'''



             lowerCamelCase__		:    Optional[int]   =  src_lang
             lowerCamelCase__		:    Dict   =  tgt_lang
             return super().prepare_seqaseq_batch(__lowerCamelCase ,				__lowerCamelCase ,				**__lowerCamelCase	)





       def 						lowerCAmelCase					(       self							:      List[Any]	):


             '''simple docstring'''



             return self.set_src_lang_special_tokens(self.src_lang	)





       def 						lowerCAmelCase					(       self							:      int	):


             '''simple docstring'''



             return self.set_tgt_lang_special_tokens(self.tgt_lang	)





       def 						lowerCAmelCase					(       self							:      Optional[int] ,				__lowerCamelCase							:      str	):


             '''simple docstring'''



             lowerCamelCase__		:    Dict   =  self.convert_tokens_to_ids(__lowerCamelCase	)
             lowerCamelCase__		:    List[Any]   =  [self.cur_lang_code_id]
             lowerCamelCase__		:    Optional[int]   =  [self.eos_token_id]

             lowerCamelCase__		:    int   =  self.convert_ids_to_tokens(self.prefix_tokens	)
             lowerCamelCase__		:    List[Any]   =  self.convert_ids_to_tokens(self.suffix_tokens	)

             lowerCamelCase__		:    List[Any]   =  processors.TemplateProcessing(
                 single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,				self.prefix_tokens + self.suffix_tokens	)	) ,				)





       def 						lowerCAmelCase					(       self							:      Optional[int] ,				__lowerCamelCase							:      str	):


             '''simple docstring'''



             lowerCamelCase__		:    int   =  self.convert_tokens_to_ids(__lowerCamelCase	)
             lowerCamelCase__		:    Optional[Any]   =  [self.cur_lang_code_id]
             lowerCamelCase__		:    Any   =  [self.eos_token_id]

             lowerCamelCase__		:    int   =  self.convert_ids_to_tokens(self.prefix_tokens	)
             lowerCamelCase__		:    Dict   =  self.convert_ids_to_tokens(self.suffix_tokens	)

             lowerCamelCase__		:    Optional[Any]   =  processors.TemplateProcessing(
                 single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,				self.prefix_tokens + self.suffix_tokens	)	) ,				)





       def 						lowerCAmelCase					(       self							:      Dict ,				__lowerCamelCase							:      Dict ,				__lowerCamelCase							:      str ,				__lowerCamelCase							:      Optional[str] ,				__lowerCamelCase							:      Optional[str] ,				**__lowerCamelCase							:      List[str]	):


             '''simple docstring'''



             if src_lang is None or tgt_lang is None:
                   raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"	)
             lowerCamelCase__		:    List[Any]   =  src_lang
             lowerCamelCase__		:    List[Any]   =  self(__lowerCamelCase ,				add_special_tokens=__lowerCamelCase ,				return_tensors=__lowerCamelCase ,				**__lowerCamelCase	)
             lowerCamelCase__		:    Dict   =  self.convert_tokens_to_ids(__lowerCamelCase	)
             lowerCamelCase__		:    Any   =  tgt_lang_id
             return inputs





       def 						lowerCAmelCase					(       self							:      List[Any] ,				__lowerCamelCase							:      str ,				__lowerCamelCase							:      Optional[str] = None	):


             '''simple docstring'''



             if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                       "tokenizer."	)

             if not os.path.isdir(__lowerCamelCase	):
                   logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
                   return
             lowerCamelCase__		:    Union[str, Any]   =  os.path.join(
                 __lowerCamelCase ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

             if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCamelCase	):
                   copyfile(self.vocab_file ,				__lowerCamelCase	)

             return (out_vocab_file,)






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin







class       _lowercase			(							lowercase__):


       """simple docstring"""




       A__ 						=      "AutoFeatureExtractor"
       A__ 						=      "Speech2Text2Tokenizer"





       def __init__(       self							:      List[str] ,				__lowerCamelCase							:      Dict ,				__lowerCamelCase							:      str	):


             '''simple docstring'''



             super().__init__(__lowerCamelCase ,				__lowerCamelCase	)
             lowerCamelCase__		:    Optional[Any]   =  self.feature_extractor
             lowerCamelCase__		:    Optional[int]   =  False





       def __call__(       self							:      List[str] ,				*__lowerCamelCase							:      Union[str, Any] ,				**__lowerCamelCase							:      List[Any]	):


             '''simple docstring'''



             if self._in_target_context_manager:
                   return self.current_processor(*__lowerCamelCase ,				**__lowerCamelCase	)

             if "raw_speech" in kwargs:
                   warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."	)
                   lowerCamelCase__		:    List[Any]   =  kwargs.pop("raw_speech"	)
             else:
                   lowerCamelCase__		:    str   =  kwargs.pop("audio" ,				__lowerCamelCase	)
             lowerCamelCase__		:    Optional[Any]   =  kwargs.pop("sampling_rate" ,				__lowerCamelCase	)
             lowerCamelCase__		:    Union[str, Any]   =  kwargs.pop("text" ,				__lowerCamelCase	)
             if len(__lowerCamelCase	) > 0:
                   lowerCamelCase__		:    Dict   =  args[0]
                   lowerCamelCase__		:    int   =  args[1:]

             if audio is None and text is None:
                   raise ValueError("You need to specify either an `audio` or `text` input to process."	)

             if audio is not None:
                   lowerCamelCase__		:    str   =  self.feature_extractor(__lowerCamelCase ,				*__lowerCamelCase ,				sampling_rate=__lowerCamelCase ,				**__lowerCamelCase	)
             if text is not None:
                   lowerCamelCase__		:    Optional[int]   =  self.tokenizer(__lowerCamelCase ,				**__lowerCamelCase	)

             if text is None:
                   return inputs
             elif audio is None:
                   return encodings
             else:
                   lowerCamelCase__		:    Optional[int]   =  encodings["input_ids"]
                   return inputs





       def 						lowerCAmelCase					(       self							:      Any ,				*__lowerCamelCase							:      Any ,				**__lowerCamelCase							:      Any	):


             '''simple docstring'''



             return self.tokenizer.batch_decode(*__lowerCamelCase ,				**__lowerCamelCase	)





       def 						lowerCAmelCase					(       self							:      Dict ,				*__lowerCamelCase							:      int ,				**__lowerCamelCase							:      List[str]	):


             '''simple docstring'''



             return self.tokenizer.decode(*__lowerCamelCase ,				**__lowerCamelCase	)





       @contextmanager
       def 						lowerCAmelCase					(       self							:      Any	):


             '''simple docstring'''



             warnings.warn(
                 "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
                 "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
                 "your audio inputs, or in a separate call."	)
             lowerCamelCase__		:    int   =  True
             lowerCamelCase__		:    List[Any]   =  self.tokenizer
             yield
             lowerCamelCase__		:    Optional[int]   =  self.feature_extractor
             lowerCamelCase__		:    List[str]   =  False


