






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
  __a :    Any								=				None


__a :    List[str]								=				logging.get_logger(__name__)

__a :    List[str]								=				{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__a :    Union[str, Any]								=				{
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"
        ),
    },
}

__a :    Dict								=				{
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}


class  __lowercase (		lowercase_					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE				  =			PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE				  =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE				  =			["input_ids", "attention_mask"]
     SCREAMING_SNAKE_CASE				  =			XLMRobertaTokenizer




     def __init__(			self    :     Tuple	,		UpperCamelCase_    :     int=None	,		UpperCamelCase_    :     List[str]=None	,		UpperCamelCase_    :     List[str]="<s>"	,		UpperCamelCase_    :     Union[str, Any]="</s>"	,		UpperCamelCase_    :     Tuple="</s>"	,		UpperCamelCase_    :     Optional[Any]="<s>"	,		UpperCamelCase_    :     List[str]="<unk>"	,		UpperCamelCase_    :     int="<pad>"	,		UpperCamelCase_    :     str="<mask>"	,		**UpperCamelCase_    :     Any	,		):


         """simple docstring"""



         __A             =		AddedToken(UpperCamelCase_	,		lstrip=UpperCamelCase_	,		rstrip=UpperCamelCase_			) if isinstance(UpperCamelCase_	,		UpperCamelCase_			) else mask_token

         super().__init__(
             UpperCamelCase_	,		tokenizer_file=UpperCamelCase_	,		bos_token=UpperCamelCase_	,		eos_token=UpperCamelCase_	,		sep_token=UpperCamelCase_	,		cls_token=UpperCamelCase_	,		unk_token=UpperCamelCase_	,		pad_token=UpperCamelCase_	,		mask_token=UpperCamelCase_	,		**UpperCamelCase_	,		)

         __A             =		vocab_file
         __A             =		False if not self.vocab_file else True




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]	,		UpperCamelCase_    :     List[int]	,		UpperCamelCase_    :     Optional[List[int]] = None			):


         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __A             =		[self.cls_token_id]
         __A             =		[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




     def 						lowerCAmelCase_       (			self    :     Dict	,		UpperCamelCase_    :     List[int]	,		UpperCamelCase_    :     Optional[List[int]] = None			):


         """simple docstring"""



         __A             =		[self.sep_token_id]
         __A             =		[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep			) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]





     def 						lowerCAmelCase_       (			self    :     Dict	,		UpperCamelCase_    :     str	,		UpperCamelCase_    :     Optional[str] = None			):


         """simple docstring"""



         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer."""			)

         if not os.path.isdir(UpperCamelCase_			):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory."			)
             return
         __A             =		os.path.join(
             UpperCamelCase_	,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

         if os.path.abspath(self.vocab_file			) != os.path.abspath(UpperCamelCase_			):
             copyfile(self.vocab_file	,		UpperCamelCase_			)

         return (out_vocab_file,)












import copy

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging


__a :    Tuple								=				logging.get_logger(__name__)


class  __lowercase (		lowercase_					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			"hybrid-clip"
     SCREAMING_SNAKE_CASE				  =			True




     def __init__(			self    :     Optional[Any]	,		UpperCamelCase_    :     Optional[Any]=512	,		**UpperCamelCase_    :     List[Any]			):


         """simple docstring"""



         super().__init__(**UpperCamelCase_			)

         if "text_config" not in kwargs:
             raise ValueError("""`text_config` can not be `None`."""			)

         if "vision_config" not in kwargs:
             raise ValueError("""`vision_config` can not be `None`."""			)

         __A             =		kwargs.pop("""text_config"""			)
         __A             =		kwargs.pop("""vision_config"""			)

         __A             =		text_config.pop("""model_type"""			)
         __A             =		vision_config.pop("""model_type"""			)

         from transformers import AutoConfig

         __A             =		AutoConfig.for_model(UpperCamelCase_	,		**UpperCamelCase_			)

         if vision_model_type == "clip":
             __A             =		AutoConfig.for_model(UpperCamelCase_	,		**UpperCamelCase_			).vision_config
         elif vision_model_type == "clip_vision_model":
             from transformers import CLIPVisionConfig

             __A             =		CLIPVisionConfig(**UpperCamelCase_			)
         else:
             __A             =		AutoConfig.for_model(UpperCamelCase_	,		**UpperCamelCase_			)

         __A             =		projection_dim
         __A             =		1.0




     @classmethod
     def 						lowerCAmelCase_       (			cls    :     int	,		UpperCamelCase_    :     PretrainedConfig	,		UpperCamelCase_    :     PretrainedConfig	,		**UpperCamelCase_    :     Union[str, Any]			):


         """simple docstring"""



         return cls(text_config=text_config.to_dict()	,		vision_config=vision_config.to_dict()	,		**UpperCamelCase_			)





     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         __A             =		copy.deepcopy(self.__dict__			)
         __A             =		self.text_config.to_dict()
         __A             =		self.vision_config.to_dict()
         __A             =		self.__class__.model_type
         return output




