

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_mbartaa import MBartaaTokenizer
else:
				_a:      str								=     None


_a:      Optional[Any]								=     logging.get_logger(__name__)

_a:      Optional[int]								=     {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_a:      List[str]								=     {
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

_a:      Optional[int]								=     {
    """facebook/mbart-large-50-one-to-many-mmt""": 1024,
}

# fmt: off
_a:      Optional[Any]								=     ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]

class   __UpperCamelCase      (   lowercase  ):
	SCREAMING_SNAKE_CASE__      			=       VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE__      			=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE__      			=       PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE__      			=       ['input_ids', 'attention_mask']
	SCREAMING_SNAKE_CASE__      			=       MBartaaTokenizer

	SCREAMING_SNAKE_CASE__      			=       []
	SCREAMING_SNAKE_CASE__      			=       []




	def __init__(						self		:  Optional[int]						,		lowerCAmelCase		:  Tuple=None						,		lowerCAmelCase		:  int=None						,		lowerCAmelCase		:  List[Any]=None						,		lowerCAmelCase		:  int=None						,		lowerCAmelCase		:  Optional[int]="</s>"						,		lowerCAmelCase		:  Union[str, Any]="</s>"						,		lowerCAmelCase		:  Any="<s>"						,		lowerCAmelCase		:  str="<unk>"						,		lowerCAmelCase		:  Union[str, Any]="<pad>"						,		lowerCAmelCase		:  List[str]="<mask>"						,		**lowerCAmelCase		:  Optional[Any]						,		):


		'''simple docstring'''







		UpperCAmelCase_    			=     AddedToken(lowerCAmelCase						,		lstrip=lowerCAmelCase						,		rstrip=lowerCAmelCase							) if isinstance(lowerCAmelCase						,		lowerCAmelCase							) else mask_token

		UpperCAmelCase_    			=     kwargs.get("additional_special_tokens"						,		[]							)
		kwargs["additional_special_tokens"] += [
		    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
		]

		super().__init__(
		    lowerCAmelCase						,		src_lang=lowerCAmelCase						,		tgt_lang=lowerCAmelCase						,		tokenizer_file=lowerCAmelCase						,		eos_token=lowerCAmelCase						,		sep_token=lowerCAmelCase						,		cls_token=lowerCAmelCase						,		unk_token=lowerCAmelCase						,		pad_token=lowerCAmelCase						,		mask_token=lowerCAmelCase						,		**lowerCAmelCase						,		)

		UpperCAmelCase_    			=     vocab_file
		UpperCAmelCase_    			=     False if not self.vocab_file else True

		UpperCAmelCase_    			=     {
		    lang_code: self.convert_tokens_to_ids(lowerCAmelCase							) for lang_code in FAIRSEQ_LANGUAGE_CODES
		}

		UpperCAmelCase_    			=     src_lang if src_lang is not None else "en_XX"
		UpperCAmelCase_    			=     tgt_lang
		UpperCAmelCase_    			=     self.lang_code_to_id[self._src_lang]
		self.set_src_lang_special_tokens(self._src_lang							)




	@property
	def  __A   (						self		:  int							):


		'''simple docstring'''







		return self._src_lang




	@src_lang.setter
	def  __A   (						self		:  List[str]						,		lowerCAmelCase		:  str							):


		'''simple docstring'''







		UpperCAmelCase_    			=     new_src_lang
		self.set_src_lang_special_tokens(self._src_lang							)




	def  __A   (						self		:  Tuple						,		lowerCAmelCase		:  List[int]						,		lowerCAmelCase		:  Optional[List[int]] = None							):


		'''simple docstring'''







		if token_ids_a is None:
			return self.prefix_tokens + token_ids_a + self.suffix_tokens
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




	def  __A   (						self		:  Dict						,		lowerCAmelCase		:  List[str]						,		lowerCAmelCase		:  str = "en_XX"						,		lowerCAmelCase		:  Optional[List[str]] = None						,		lowerCAmelCase		:  str = "ro_RO"						,		**lowerCAmelCase		:  List[str]						,		):


		'''simple docstring'''







		UpperCAmelCase_    			=     src_lang
		UpperCAmelCase_    			=     tgt_lang
		return super().prepare_seqaseq_batch(lowerCAmelCase						,		lowerCAmelCase						,		**lowerCAmelCase							)




	def  __A   (						self		:  List[str]							):


		'''simple docstring'''







		return self.set_src_lang_special_tokens(self.src_lang							)




	def  __A   (						self		:  Optional[int]							):


		'''simple docstring'''







		return self.set_tgt_lang_special_tokens(self.tgt_lang							)




	def  __A   (						self		:  Optional[int]						,		lowerCAmelCase		:  str							):


		'''simple docstring'''







		UpperCAmelCase_    			=     self.convert_tokens_to_ids(lowerCAmelCase							)
		UpperCAmelCase_    			=     [self.cur_lang_code_id]
		UpperCAmelCase_    			=     [self.eos_token_id]

		UpperCAmelCase_    			=     self.convert_ids_to_tokens(self.prefix_tokens							)
		UpperCAmelCase_    			=     self.convert_ids_to_tokens(self.suffix_tokens							)

		UpperCAmelCase_    			=     processors.TemplateProcessing(
		    single=prefix_tokens_str + ["$A"] + suffix_tokens_str						,		pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str						,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,		self.prefix_tokens + self.suffix_tokens							)							)						,		)




	def  __A   (						self		:  str						,		lowerCAmelCase		:  str							):


		'''simple docstring'''







		UpperCAmelCase_    			=     self.convert_tokens_to_ids(lowerCAmelCase							)
		UpperCAmelCase_    			=     [self.cur_lang_code_id]
		UpperCAmelCase_    			=     [self.eos_token_id]

		UpperCAmelCase_    			=     self.convert_ids_to_tokens(self.prefix_tokens							)
		UpperCAmelCase_    			=     self.convert_ids_to_tokens(self.suffix_tokens							)

		UpperCAmelCase_    			=     processors.TemplateProcessing(
		    single=prefix_tokens_str + ["$A"] + suffix_tokens_str						,		pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str						,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,		self.prefix_tokens + self.suffix_tokens							)							)						,		)




	def  __A   (						self		:  str						,		lowerCAmelCase		:  Any						,		lowerCAmelCase		:  str						,		lowerCAmelCase		:  Optional[str]						,		lowerCAmelCase		:  Optional[str]						,		**lowerCAmelCase		:  Union[str, Any]							):


		'''simple docstring'''







		if src_lang is None or tgt_lang is None:
			raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"							)
		UpperCAmelCase_    			=     src_lang
		UpperCAmelCase_    			=     self(lowerCAmelCase						,		add_special_tokens=lowerCAmelCase						,		return_tensors=lowerCAmelCase						,		**lowerCAmelCase							)
		UpperCAmelCase_    			=     self.convert_tokens_to_ids(lowerCAmelCase							)
		UpperCAmelCase_    			=     tgt_lang_id
		return inputs




	def  __A   (						self		:  Optional[Any]						,		lowerCAmelCase		:  str						,		lowerCAmelCase		:  Optional[str] = None							):


		'''simple docstring'''







		if not self.can_save_slow_tokenizer:
			raise ValueError(
			    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
			    "tokenizer."							)

		if not os.path.isdir(lowerCAmelCase							):
			logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
			return
		UpperCAmelCase_    			=     os.path.join(
		    lowerCAmelCase						,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

		if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCAmelCase							):
			copyfile(self.vocab_file						,		lowerCAmelCase							)

		return (out_vocab_file,)






import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
				import torch_xla.core.xla_model as xm  # noqa: F401

				_a:      List[Any]								=     True
except ImportError:
				_a:      int								=     False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_a:      str								=     torch.distributed.is_available()



def  __lowerCAmelCase (  A      ):
	# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
	UpperCAmelCase_    			=     importlib.util.find_spec(A      ) is not None
	if package_exists:
		try:
			UpperCAmelCase_    			=     importlib.metadata.metadata(A      )
			return True
		except importlib.metadata.PackageNotFoundError:
			return False



def  __lowerCAmelCase (  ):
	return _torch_distributed_available



def  __lowerCAmelCase (  ):
	try:
		pass
	except ImportError:
		print(
		    "Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not"
		    " detected. If you see \"ValueError: Invalid backend: 'ccl'\" error, please install Intel(R) oneCCL"
		    " Bindings for PyTorch*."      )
	return (
	    importlib.util.find_spec("torch_ccl"      ) is not None
	    or importlib.util.find_spec("oneccl_bindings_for_pytorch"      ) is not None
	)



def  __lowerCAmelCase (  ):
	return importlib.metadata.version("oneccl_bind_pt"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("transformer_engine"      )



@lru_cache
def  __lowerCAmelCase (  A=True      ):
	# Due to bugs on the amp series GPUs, we disable torch-xla on them
	if torch.cuda.is_available():
		return False
	if _tpu_available and check_device:
		try:
			# Will raise a RuntimeError if no XLA configuration is found
			UpperCAmelCase_    			=     xm.xla_device()
			return True
		except RuntimeError:
			return False
	return _tpu_available



def  __lowerCAmelCase (  ):
	return _is_package_available("deepspeed"      )



def  __lowerCAmelCase (  A=False      ):
	if is_tpu_available():
		return not ignore_tpu
	if torch.cuda.is_available():
		return torch.cuda.is_bfaa_supported()
	if is_npu_available():
		return False
	return True



def  __lowerCAmelCase (  ):
	UpperCAmelCase_    			=     _is_package_available("bitsandbytes"      )
	if package_exists:
		UpperCAmelCase_    			=     version.parse(importlib.metadata.version("bitsandbytes"      )      )
		return compare_versions(A    ,       ">="    ,       "0.39.0"      )
	return False



def  __lowerCAmelCase (  ):
	UpperCAmelCase_    			=     _is_package_available("bitsandbytes"      )
	if package_exists:
		UpperCAmelCase_    			=     version.parse(importlib.metadata.version("bitsandbytes"      )      )
		return compare_versions(A    ,       ">="    ,       "0.37.2"      )
	return False



def  __lowerCAmelCase (  ):
	return _is_package_available("bitsandbytes"      )



def  __lowerCAmelCase (  ):
	if strtobool(os.environ.get("ACCELERATE_USE_MEGATRON_LM"    ,       "False"      )      ) == 1:
		UpperCAmelCase_    			=     importlib.util.find_spec("megatron"      ) is not None
		if package_exists:
			try:
				UpperCAmelCase_    			=     parse(importlib.metadata.version("megatron-lm"      )      )
				return compare_versions(A    ,       ">="    ,       "2.2.0"      )
			except Exception as e:
				warnings.warn(F"Parse Megatron version failed. Exception:{e}"      )
				return False



def  __lowerCAmelCase (  ):
	return _is_package_available("safetensors"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("transformers"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("datasets"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("aim"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("tensorboard"      ) or _is_package_available("tensorboardX"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("wandb"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("comet_ml"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("boto3"      )



def  __lowerCAmelCase (  ):
	if _is_package_available("rich"      ):
		if "ACCELERATE_DISABLE_RICH" in os.environ:
			warnings.warn(
			    "`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`."      )
			return not parse_flag_from_env("ACCELERATE_DISABLE_RICH"    ,       A      )
		return parse_flag_from_env("ACCELERATE_ENABLE_RICH"    ,       A      )
	return False



def  __lowerCAmelCase (  ):
	return _is_package_available("sagemaker"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("tqdm"      )



def  __lowerCAmelCase (  ):
	return _is_package_available("mlflow"      )



def  __lowerCAmelCase (  ):
	return is_torch_version(">="    ,       "1.12"      ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()



def  __lowerCAmelCase (  ):
	def get_major_and_minor_from_version(A      ):
		return str(version.parse(A      ).major      ) + "." + str(version.parse(A      ).minor      )

	UpperCAmelCase_    			=     importlib.metadata.version("torch"      )
	if importlib.util.find_spec("intel_extension_for_pytorch"      ) is None:
		return False
	UpperCAmelCase_    			=     "N/A"
	try:
		UpperCAmelCase_    			=     importlib.metadata.version("intel_extension_for_pytorch"      )
	except importlib.metadata.PackageNotFoundError:
		return False
	UpperCAmelCase_    			=     get_major_and_minor_from_version(_torch_version      )
	UpperCAmelCase_    			=     get_major_and_minor_from_version(_ipex_version      )
	if torch_major_and_minor != ipex_major_and_minor:
		warnings.warn(
		    F"Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"
		    F" but PyTorch {_torch_version} is found. Please switch to the matching version and run again."      )
		return False
	return True



@lru_cache
def  __lowerCAmelCase (  A=False      ):
	if importlib.util.find_spec("torch"      ) is None or importlib.util.find_spec("torch_npu"      ) is None:
		return False

	import torch
	import torch_npu  # noqa: F401

	if check_device:
		try:
			# Will raise a RuntimeError if no NPU is found
			UpperCAmelCase_    			=     torch.npu.device_count()
			return torch.npu.is_available()
		except RuntimeError:
			return False
	return hasattr(A    ,       "npu"      ) and torch.npu.is_available()




@lru_cache
def  __lowerCAmelCase (  A=False      ):
	if not parse_flag_from_env("ACCELERATE_USE_XPU"    ,       default=A      ):
		return False
	if is_ipex_available():
		import torch

		if is_torch_version("<="    ,       "1.12"      ):
			return False
	else:
		return False

	import intel_extension_for_pytorch  # noqa: F401

	if check_device:
		try:
			# Will raise a RuntimeError if no XPU  is found
			UpperCAmelCase_    			=     torch.xpu.device_count()
			return torch.xpu.is_available()
		except RuntimeError:
			return False
	return hasattr(A    ,       "xpu"      ) and torch.xpu.is_available()