






import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a				     =  logging.get_logger(__name__)
_a				     =  {'''vocab_file''': '''spiece.model'''}

_a				     =  {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
}

_a				     =  {
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

_a				     =  '''▁'''





class __lowerCamelCase			(				snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				VOCAB_FILES_NAMES
	UpperCamelCase__								=				PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase__								=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase__								=				['input_ids', 'token_type_ids']

	def __init__(							self	,   UpperCAmelCase	,   UpperCAmelCase=False	,   UpperCAmelCase=True	,   UpperCAmelCase=True	,   UpperCAmelCase="<unk>"	,   UpperCAmelCase="[SEP]"	,   UpperCAmelCase="<pad>"	,   UpperCAmelCase="[CLS]"	,   UpperCAmelCase="[MASK]"	,   UpperCAmelCase = None	,   **UpperCAmelCase	,   ):
							"""simple docstring"""







							_UpperCAmelCase     =      (
							    AddedToken(UpperCAmelCase	,   lstrip=UpperCAmelCase	,   rstrip=UpperCAmelCase	,   normalized=UpperCAmelCase   )
							    if isinstance(UpperCAmelCase	,   UpperCAmelCase   )
							    else mask_token
							)

							_UpperCAmelCase     =      {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=UpperCAmelCase	,   remove_space=UpperCAmelCase	,   keep_accents=UpperCAmelCase	,   unk_token=UpperCAmelCase	,   sep_token=UpperCAmelCase	,   pad_token=UpperCAmelCase	,   cls_token=UpperCAmelCase	,   mask_token=UpperCAmelCase	,   sp_model_kwargs=self.sp_model_kwargs	,   **UpperCAmelCase	,   )

							_UpperCAmelCase     =      do_lower_case
							_UpperCAmelCase     =      remove_space
							_UpperCAmelCase     =      keep_accents
							_UpperCAmelCase     =      vocab_file

							_UpperCAmelCase     =      spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(UpperCAmelCase   )

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return len(self.sp_model   )

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      {self.convert_ids_to_tokens(UpperCAmelCase   ): i for i in range(self.vocab_size   )}
							vocab.update(self.added_tokens_encoder   )
							return vocab

	def __getstate__(							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      self.__dict__.copy()
							_UpperCAmelCase     =      None
							return state

	def __setstate__(							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      d

							# for backward compatibility
							if not hasattr(self	,   'sp_model_kwargs'   ):
													_UpperCAmelCase     =      {}

							_UpperCAmelCase     =      spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(self.vocab_file   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							if self.remove_space:
													_UpperCAmelCase     =      ' '.join(inputs.strip().split()   )
							else:
													_UpperCAmelCase     =      inputs
							_UpperCAmelCase     =      outputs.replace('``'	,   '"'   ).replace('\'\''	,   '"'   )

							if not self.keep_accents:
													_UpperCAmelCase     =      unicodedata.normalize('NFKD'	,   UpperCAmelCase   )
													_UpperCAmelCase     =      ''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase   )]   )
							if self.do_lower_case:
													_UpperCAmelCase     =      outputs.lower()

							return outputs

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      self.preprocess_text(UpperCAmelCase   )
							_UpperCAmelCase     =      self.sp_model.encode(UpperCAmelCase	,   out_type=UpperCAmelCase   )
							_UpperCAmelCase     =      []
							for piece in pieces:
													if len(UpperCAmelCase   ) > 1 and piece[-1] == str(','   ) and piece[-2].isdigit():
																			_UpperCAmelCase     =      self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase	,   ''   )   )
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																									if len(cur_pieces[0]   ) == 1:
																															_UpperCAmelCase     =      cur_pieces[1:]
																									else:
																															_UpperCAmelCase     =      cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]   )
																			new_pieces.extend(UpperCAmelCase   )
													else:
																			new_pieces.append(UpperCAmelCase   )

							return new_pieces

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							return self.sp_model.PieceToId(UpperCAmelCase   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							return self.sp_model.IdToPiece(UpperCAmelCase   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      []
							_UpperCAmelCase     =      ''
							_UpperCAmelCase     =      False
							for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																			if not prev_is_special:
																									out_string += " "
																			out_string += self.sp_model.decode(UpperCAmelCase   ) + token
																			_UpperCAmelCase     =      True
																			_UpperCAmelCase     =      []
													else:
																			current_sub_tokens.append(UpperCAmelCase   )
																			_UpperCAmelCase     =      False
							out_string += self.sp_model.decode(UpperCAmelCase   )
							return out_string.strip()

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = False	,   UpperCAmelCase = None	,   UpperCAmelCase = True	,   **UpperCAmelCase	,   ):
							"""simple docstring"""







							_UpperCAmelCase     =      kwargs.pop('use_source_tokenizer'	,   UpperCAmelCase   )

							_UpperCAmelCase     =      self.convert_ids_to_tokens(UpperCAmelCase	,   skip_special_tokens=UpperCAmelCase   )

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							_UpperCAmelCase     =      []
							_UpperCAmelCase     =      []
							for token in filtered_tokens:
													if skip_special_tokens and token in self.all_special_ids:
																			continue
													if token in self.added_tokens_encoder:
																			if current_sub_text:
																									sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase   )   )
																									_UpperCAmelCase     =      []
																			sub_texts.append(UpperCAmelCase   )
													else:
																			current_sub_text.append(UpperCAmelCase   )
							if current_sub_text:
													sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase   )   )

							# Mimic the behavior of the Rust tokenizer:
							# No space after <unk>
							if spaces_between_special_tokens:
													_UpperCAmelCase     =      re.sub(R'(<unk>) '	,   R'\1'	,   ' '.join(UpperCAmelCase   )   )
							else:
													_UpperCAmelCase     =      ''.join(UpperCAmelCase   )

							_UpperCAmelCase     =      (
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
													_UpperCAmelCase     =      self.clean_up_tokenization(UpperCAmelCase   )
													return clean_text
							else:
													return text

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							_UpperCAmelCase     =      [self.sep_token_id]
							_UpperCAmelCase     =      [self.cls_token_id]
							if token_ids_a is None:
													return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None	,   UpperCAmelCase = False   ):
							"""simple docstring"""







							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=UpperCAmelCase	,   token_ids_a=UpperCAmelCase	,   already_has_special_tokens=UpperCAmelCase   )

							if token_ids_a is not None:
													return [1] + ([0] * len(UpperCAmelCase   )) + [1] + ([0] * len(UpperCAmelCase   )) + [1]
							return [1] + ([0] * len(UpperCAmelCase   )) + [1]

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							_UpperCAmelCase     =      [self.sep_token_id]
							_UpperCAmelCase     =      [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							if not os.path.isdir(UpperCAmelCase   ):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
													return
							_UpperCAmelCase     =      os.path.join(
							    UpperCAmelCase	,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

							if os.path.abspath(self.vocab_file   ) != os.path.abspath(UpperCAmelCase   ) and os.path.isfile(self.vocab_file   ):
													copyfile(self.vocab_file	,   UpperCAmelCase   )
							elif not os.path.isfile(self.vocab_file   ):
													with open(UpperCAmelCase	,   'wb'   ) as fi:
																			_UpperCAmelCase     =      self.sp_model.serialized_model_proto()
																			fi.write(UpperCAmelCase   )

							return (out_vocab_file,)








from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a				     =  logging.get_logger(__name__)

_a				     =  {
    '''facebook/regnet-y-040''': '''https://huggingface.co/facebook/regnet-y-040/blob/main/config.json''',
}





class __lowerCamelCase			(				snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				"regnet"
	UpperCamelCase__								=				["x", "y"]

	def __init__(							self	,   UpperCAmelCase=3	,   UpperCAmelCase=32	,   UpperCAmelCase=[128, 192, 512, 1088]	,   UpperCAmelCase=[2, 6, 12, 2]	,   UpperCAmelCase=64	,   UpperCAmelCase="y"	,   UpperCAmelCase="relu"	,   **UpperCAmelCase	,   ):
							"""simple docstring"""







							super().__init__(**UpperCAmelCase   )
							if layer_type not in self.layer_types:
													raise ValueError(F"""layer_type={layer_type} is not one of {",".join(self.layer_types   )}"""   )
							_UpperCAmelCase     =      num_channels
							_UpperCAmelCase     =      embedding_size
							_UpperCAmelCase     =      hidden_sizes
							_UpperCAmelCase     =      depths
							_UpperCAmelCase     =      groups_width
							_UpperCAmelCase     =      layer_type
							_UpperCAmelCase     =      hidden_act
							# always downsample in the first stage
							_UpperCAmelCase     =      True
