


import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase						 =	logging.get_logger(__name__)
lowerCamelCase						 =	{'vocab_file': 'spiece.model'}

lowerCamelCase						 =	{
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
}

lowerCamelCase						 =	{
    'google/fnet-base': 5_12,
    'google/fnet-large': 5_12,
}

lowerCamelCase						 =	'▁'




class A  (       UpperCamelCase_					):
      UpperCamelCase__							:  Any				=						VOCAB_FILES_NAMES
      UpperCamelCase__							:  int				=						PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__							:  Optional[int]				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__							:  Tuple				=						['input_ids', 'token_type_ids']






      def __init__( self						:					Dict      ,       lowercase_						:					Optional[Any]      ,       lowercase_						:					Any=False      ,       lowercase_						:					Any=True      ,       lowercase_						:					Tuple=True      ,       lowercase_						:					List[str]="<unk>"      ,       lowercase_						:					List[Any]="[SEP]"      ,       lowercase_						:					Optional[int]="<pad>"      ,       lowercase_						:					List[Any]="[CLS]"      ,       lowercase_						:					Any="[MASK]"      ,       lowercase_						:					List[str] = None      ,       **lowercase_						:					Optional[int]      ,       ) ->						None:
             """simple docstring"""




             _lowerCamelCase	:		int          =(
                 AddedToken(lowercase_      ,       lstrip=lowercase_      ,       rstrip=lowercase_      ,       normalized=lowercase_			)
                 if isinstance(lowercase_      ,       lowercase_			)
                 else mask_token
             )

             _lowerCamelCase	:		Any          ={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=lowercase_      ,       remove_space=lowercase_      ,       keep_accents=lowercase_      ,       unk_token=lowercase_      ,       sep_token=lowercase_      ,       pad_token=lowercase_      ,       cls_token=lowercase_      ,       mask_token=lowercase_      ,       sp_model_kwargs=self.sp_model_kwargs      ,       **lowercase_      ,       )

             _lowerCamelCase	:		Dict          =do_lower_case
             _lowerCamelCase	:		List[Any]          =remove_space
             _lowerCamelCase	:		Any          =keep_accents
             _lowerCamelCase	:		Union[str, Any]          =vocab_file

             _lowerCamelCase	:		str          =spm.SentencePieceProcessor(**self.sp_model_kwargs			)
             self.sp_model.Load(lowercase_			)






      @property
      def 	lowerCamelCase	( self						:					List[str]			) ->						List[str]:
             """simple docstring"""




             return len(self.sp_model			)






      def 	lowerCamelCase	( self						:					Dict			) ->						List[Any]:
             """simple docstring"""




             _lowerCamelCase	:		Any          ={self.convert_ids_to_tokens(lowercase_			): i for i in range(self.vocab_size			)}
             vocab.update(self.added_tokens_encoder			)
             return vocab






      def __getstate__( self						:					Optional[Any]			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		Dict          =self.__dict__.copy()
             _lowerCamelCase	:		List[str]          =None
             return state






      def __setstate__( self						:					Dict      ,       lowercase_						:					Optional[int]			) ->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[Any]          =d

             # for backward compatibility
             if not hasattr(self      ,       'sp_model_kwargs'			):
                    _lowerCamelCase	:		Optional[int]          ={}

             _lowerCamelCase	:		List[str]          =spm.SentencePieceProcessor(**self.sp_model_kwargs			)
             self.sp_model.Load(self.vocab_file			)






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					str			) ->						Dict:
             """simple docstring"""




             if self.remove_space:
                    _lowerCamelCase	:		Optional[int]          =' '.join(inputs.strip().split()			)
             else:
                    _lowerCamelCase	:		List[Any]          =inputs
             _lowerCamelCase	:		Union[str, Any]          =outputs.replace('``'      ,       '"'			).replace('\'\''      ,       '"'			)

             if not self.keep_accents:
                    _lowerCamelCase	:		str          =unicodedata.normalize('NFKD'      ,       lowercase_			)
                    _lowerCamelCase	:		Optional[Any]          =''.join([c for c in outputs if not unicodedata.combining(lowercase_			)]			)
             if self.do_lower_case:
                    _lowerCamelCase	:		Optional[Any]          =outputs.lower()

             return outputs






      def 	lowerCamelCase	( self						:					List[Any]      ,       lowercase_						:					Tuple			) ->						List[str]:
             """simple docstring"""




             _lowerCamelCase	:		Optional[int]          =self.preprocess_text(lowercase_			)
             _lowerCamelCase	:		Dict          =self.sp_model.encode(lowercase_      ,       out_type=lowercase_			)
             _lowerCamelCase	:		Tuple          =[]
             for piece in pieces:
                    if len(lowercase_			) > 1 and piece[-1] == str(','			) and piece[-2].isdigit():
                           _lowerCamelCase	:		Tuple          =self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_      ,       ''			)			)
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]			) == 1:
                                         _lowerCamelCase	:		Dict          =cur_pieces[1:]
                                  else:
                                         _lowerCamelCase	:		Union[str, Any]          =cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]			)
                           new_pieces.extend(lowercase_			)
                    else:
                           new_pieces.append(lowercase_			)

             return new_pieces






      def 	lowerCamelCase	( self						:					Dict      ,       lowercase_						:					List[str]			) ->						Dict:
             """simple docstring"""




             return self.sp_model.PieceToId(lowercase_			)






      def 	lowerCamelCase	( self						:					Tuple      ,       lowercase_						:					int			) ->						Union[str, Any]:
             """simple docstring"""




             return self.sp_model.IdToPiece(lowercase_			)






      def 	lowerCamelCase	( self						:					str      ,       lowercase_						:					Dict			) ->						Any:
             """simple docstring"""




             _lowerCamelCase	:		Dict          =[]
             _lowerCamelCase	:		Any          =''
             _lowerCamelCase	:		Any          =False
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           if not prev_is_special:
                                  out_string += " "
                           out_string += self.sp_model.decode(lowercase_			) + token
                           _lowerCamelCase	:		Any          =True
                           _lowerCamelCase	:		Any          =[]
                    else:
                           current_sub_tokens.append(lowercase_			)
                           _lowerCamelCase	:		List[Any]          =False
             out_string += self.sp_model.decode(lowercase_			)
             return out_string.strip()






      def 	lowerCamelCase	( self						:					int      ,       lowercase_						:					Union[str, Any]      ,       lowercase_						:					Dict = False      ,       lowercase_						:					str = None      ,       lowercase_						:					int = True      ,       **lowercase_						:					int      ,       ) ->						str:
             """simple docstring"""




             _lowerCamelCase	:		Any          =kwargs.pop('use_source_tokenizer'      ,       lowercase_			)

             _lowerCamelCase	:		List[Any]          =self.convert_ids_to_tokens(lowercase_      ,       skip_special_tokens=lowercase_			)

             # To avoid mixing byte-level and unicode for byte-level BPT
             # we need to build string separately for added tokens and byte-level tokens
             # cf. https://github.com/huggingface/transformers/issues/1133
             _lowerCamelCase	:		List[Any]          =[]
             _lowerCamelCase	:		str          =[]
             for token in filtered_tokens:
                    if skip_special_tokens and token in self.all_special_ids:
                           continue
                    if token in self.added_tokens_encoder:
                           if current_sub_text:
                                  sub_texts.append(self.convert_tokens_to_string(lowercase_			)			)
                                  _lowerCamelCase	:		List[Any]          =[]
                           sub_texts.append(lowercase_			)
                    else:
                           current_sub_text.append(lowercase_			)
             if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(lowercase_			)			)

             # Mimic the behavior of the Rust tokenizer:
             # No space after <unk>
             if spaces_between_special_tokens:
                    _lowerCamelCase	:		Union[str, Any]          =re.sub(R'(<unk>) '      ,       R'\1'      ,       ' '.join(lowercase_			)			)
             else:
                    _lowerCamelCase	:		Tuple          =''.join(lowercase_			)

             _lowerCamelCase	:		int          =(
                 clean_up_tokenization_spaces
                 if clean_up_tokenization_spaces is not None
                 else self.clean_up_tokenization_spaces
             )
             if clean_up_tokenization_spaces:
                    _lowerCamelCase	:		Any          =self.clean_up_tokenization(lowercase_			)
                    return clean_text
             else:
                    return text






      def 	lowerCamelCase	( self						:					Dict      ,       lowercase_						:					int      ,       lowercase_						:					Union[str, Any] = None			) ->						List[int]:
             """simple docstring"""




             _lowerCamelCase	:		Any          =[self.sep_token_id]
             _lowerCamelCase	:		List[str]          =[self.cls_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep






      def 	lowerCamelCase	( self						:					Optional[Any]      ,       lowercase_						:					List[Any]      ,       lowercase_						:					Optional[int] = None      ,       lowercase_						:					List[Any] = False			) ->						List[int]:
             """simple docstring"""




             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=lowercase_      ,       token_ids_a=lowercase_      ,       already_has_special_tokens=lowercase_			)

             if token_ids_a is not None:
                    return [1] + ([0] * len(lowercase_			)) + [1] + ([0] * len(lowercase_			)) + [1]
             return [1] + ([0] * len(lowercase_			)) + [1]






      def 	lowerCamelCase	( self						:					int      ,       lowercase_						:					List[Any]      ,       lowercase_						:					str = None			) ->						List[int]:
             """simple docstring"""




             _lowerCamelCase	:		Any          =[self.sep_token_id]
             _lowerCamelCase	:		Dict          =[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep			) * [0]
             return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]






      def 	lowerCamelCase	( self						:					Union[str, Any]      ,       lowercase_						:					int      ,       lowercase_						:					Optional[int] = None			) ->						Tuple[str]:
             """simple docstring"""




             if not os.path.isdir(lowercase_			):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
                    return
             _lowerCamelCase	:		Any          =os.path.join(
                 lowercase_      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

             if os.path.abspath(self.vocab_file			) != os.path.abspath(lowercase_			) and os.path.isfile(self.vocab_file			):
                    copyfile(self.vocab_file      ,       lowercase_			)
             elif not os.path.isfile(self.vocab_file			):
                    with open(lowercase_      ,       'wb'			) as fi:
                           _lowerCamelCase	:		Any          =self.sp_model.serialized_model_proto()
                           fi.write(lowercase_			)

             return (out_vocab_file,)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase						 =	{'configuration_visual_bert': ['VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VisualBertConfig']}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase						 =	[
     'VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'VisualBertForMultipleChoice',
     'VisualBertForPreTraining',
     'VisualBertForQuestionAnswering',
     'VisualBertForRegionToPhraseAlignment',
     'VisualBertForVisualReasoning',
     'VisualBertLayer',
     'VisualBertModel',
     'VisualBertPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_visual_bert import (
      VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      VisualBertForMultipleChoice,
      VisualBertForPreTraining,
      VisualBertForQuestionAnswering,
      VisualBertForRegionToPhraseAlignment,
      VisualBertForVisualReasoning,
      VisualBertLayer,
      VisualBertModel,
      VisualBertPreTrainedModel,
  )


else:
 import sys

 lowerCamelCase						 =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


