

'''simple docstring'''






import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_ :    Any					  =	logging.get_logger(__name__)
snake_case_ :    Optional[Any]					  =	{"vocab_file": "spiece.model"}

snake_case_ :    Optional[int]					  =	{
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
}

snake_case_ :    str					  =	{
    "google/fnet-base": 5_1_2,
    "google/fnet-large": 5_1_2,
}

snake_case_ :    Optional[Any]					  =	"▁"
class      SCREAMING_SNAKE_CASE__      (    lowerCAmelCase_				):
    _A         =							VOCAB_FILES_NAMES
    _A         =							PRETRAINED_VOCAB_FILES_MAP
    _A         =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A         =							["input_ids", "token_type_ids"]




    def __init__( self    ,					lowercase__    ,					lowercase__=False    ,					lowercase__=True    ,					lowercase__=True    ,					lowercase__="<unk>"    ,					lowercase__="[SEP]"    ,					lowercase__="<pad>"    ,					lowercase__="[CLS]"    ,					lowercase__="[MASK]"    ,					lowercase__ = None    ,					**lowercase__    ,					):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      List[str]					=	(
              AddedToken(_snake_case    ,					lstrip=_snake_case    ,					rstrip=_snake_case    ,					normalized=_snake_case      )
              if isinstance(_snake_case    ,					_snake_case      )
              else mask_token
          )

          SCREAMING_SNAKE_CASE_				:      Any					=	{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=_snake_case    ,					remove_space=_snake_case    ,					keep_accents=_snake_case    ,					unk_token=_snake_case    ,					sep_token=_snake_case    ,					pad_token=_snake_case    ,					cls_token=_snake_case    ,					mask_token=_snake_case    ,					sp_model_kwargs=self.sp_model_kwargs    ,					**_snake_case    ,					)

          SCREAMING_SNAKE_CASE_				:      Dict					=	do_lower_case
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	remove_space
          SCREAMING_SNAKE_CASE_				:      str					=	keep_accents
          SCREAMING_SNAKE_CASE_				:      Dict					=	vocab_file

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
          self.sp_model.Load(_snake_case      )




    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return len(self.sp_model      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Tuple					=	{self.convert_ids_to_tokens(_snake_case      ): i for i in range(self.vocab_size      )}
          vocab.update(self.added_tokens_encoder      )
          return vocab




    def __getstate__( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	self.__dict__.copy()
          SCREAMING_SNAKE_CASE_				:      int					=	None
          return state




    def __setstate__( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Dict					=	d

          # for backward compatibility
          if not hasattr(self    ,					"sp_model_kwargs"      ):
                SCREAMING_SNAKE_CASE_				:      Dict					=	{}

          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
          self.sp_model.Load(self.vocab_file      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          if self.remove_space:
                SCREAMING_SNAKE_CASE_				:      Dict					=	' '.join(inputs.strip().split()      )
          else:
                SCREAMING_SNAKE_CASE_				:      Any					=	inputs
          SCREAMING_SNAKE_CASE_				:      List[str]					=	outputs.replace("``"    ,					"\""      ).replace("\'\'"    ,					"\""      )

          if not self.keep_accents:
                SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	unicodedata.normalize("NFKD"    ,					_snake_case      )
                SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	''.join([c for c in outputs if not unicodedata.combining(_snake_case      )]      )
          if self.do_lower_case:
                SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	outputs.lower()

          return outputs




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      List[str]					=	self.preprocess_text(_snake_case      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.sp_model.encode(_snake_case    ,					out_type=_snake_case      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	[]
          for piece in pieces:
                if len(_snake_case      ) > 1 and piece[-1] == str(","      ) and piece[-2].isdigit():
                      SCREAMING_SNAKE_CASE_				:      Dict					=	self.sp_model.EncodeAsPieces(piece[:-1].replace(_snake_case    ,					""      )      )
                      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                            if len(cur_pieces[0]      ) == 1:
                                  SCREAMING_SNAKE_CASE_				:      List[Any]					=	cur_pieces[1:]
                            else:
                                  SCREAMING_SNAKE_CASE_				:      int					=	cur_pieces[0][1:]
                      cur_pieces.append(piece[-1]      )
                      new_pieces.extend(_snake_case      )
                else:
                      new_pieces.append(_snake_case      )

          return new_pieces




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          return self.sp_model.PieceToId(_snake_case      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          return self.sp_model.IdToPiece(_snake_case      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	[]
          SCREAMING_SNAKE_CASE_				:      int					=	''
          SCREAMING_SNAKE_CASE_				:      int					=	False
          for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                      if not prev_is_special:
                            out_string += " "
                      out_string += self.sp_model.decode(_snake_case      ) + token
                      SCREAMING_SNAKE_CASE_				:      str					=	True
                      SCREAMING_SNAKE_CASE_				:      int					=	[]
                else:
                      current_sub_tokens.append(_snake_case      )
                      SCREAMING_SNAKE_CASE_				:      List[str]					=	False
          out_string += self.sp_model.decode(_snake_case      )
          return out_string.strip()




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = False    ,					lowercase__ = None    ,					lowercase__ = True    ,					**lowercase__    ,					):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	kwargs.pop("use_source_tokenizer"    ,					_snake_case      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	self.convert_ids_to_tokens(_snake_case    ,					skip_special_tokens=_snake_case      )

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          SCREAMING_SNAKE_CASE_				:      List[str]					=	[]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[]
          for token in filtered_tokens:
                if skip_special_tokens and token in self.all_special_ids:
                      continue
                if token in self.added_tokens_encoder:
                      if current_sub_text:
                            sub_texts.append(self.convert_tokens_to_string(_snake_case      )      )
                            SCREAMING_SNAKE_CASE_				:      int					=	[]
                      sub_texts.append(_snake_case      )
                else:
                      current_sub_text.append(_snake_case      )
          if current_sub_text:
                sub_texts.append(self.convert_tokens_to_string(_snake_case      )      )

          # Mimic the behavior of the Rust tokenizer:
          # No space after <unk>
          if spaces_between_special_tokens:
                SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	re.sub(r"(<unk>) "    ,					r"\1"    ,					" ".join(_snake_case      )      )
          else:
                SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	''.join(_snake_case      )

          SCREAMING_SNAKE_CASE_				:      Any					=	(
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
                SCREAMING_SNAKE_CASE_				:      Any					=	self.clean_up_tokenization(_snake_case      )
                return clean_text
          else:
                return text




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Dict					=	[self.sep_token_id]
          SCREAMING_SNAKE_CASE_				:      List[str]					=	[self.cls_token_id]
          if token_ids_a is None:
                return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None    ,					lowercase__ = False      ):






          """simple docstring"""






          if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=_snake_case    ,					token_ids_a=_snake_case    ,					already_has_special_tokens=_snake_case      )

          if token_ids_a is not None:
                return [1] + ([0] * len(_snake_case      )) + [1] + ([0] * len(_snake_case      )) + [1]
          return [1] + ([0] * len(_snake_case      )) + [1]




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	[self.sep_token_id]
          SCREAMING_SNAKE_CASE_				:      List[str]					=	[self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep      ) * [0]
          return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None      ):






          """simple docstring"""






          if not os.path.isdir(_snake_case      ):
                logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
                return
          SCREAMING_SNAKE_CASE_				:      Dict					=	os.path.join(
              _snake_case    ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

          if os.path.abspath(self.vocab_file      ) != os.path.abspath(_snake_case      ) and os.path.isfile(self.vocab_file      ):
                copyfile(self.vocab_file    ,					_snake_case      )
          elif not os.path.isfile(self.vocab_file      ):
                with open(_snake_case    ,					"wb"      ) as fi:
                      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	self.sp_model.serialized_model_proto()
                      fi.write(_snake_case      )

          return (out_vocab_file,)




'''simple docstring'''






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_					  =	logging.get_logger(__name__)

snake_case_					  =	{
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase				):
    _A         =							"hubert"




    def __init__( self    ,					lowercase__=32    ,					lowercase__=768    ,					lowercase__=12    ,					lowercase__=12    ,					lowercase__=3072    ,					lowercase__="gelu"    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=True    ,					lowercase__=0.0    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.02    ,					lowercase__=1e-5    ,					lowercase__="group"    ,					lowercase__="gelu"    ,					lowercase__=(512, 512, 512, 512, 512, 512, 512)    ,					lowercase__=(5, 2, 2, 2, 2, 2, 2)    ,					lowercase__=(10, 3, 3, 3, 3, 2, 2)    ,					lowercase__=False    ,					lowercase__=128    ,					lowercase__=16    ,					lowercase__=False    ,					lowercase__=True    ,					lowercase__=0.05    ,					lowercase__=10    ,					lowercase__=2    ,					lowercase__=0.0    ,					lowercase__=10    ,					lowercase__=0    ,					lowercase__="sum"    ,					lowercase__=False    ,					lowercase__=False    ,					lowercase__=256    ,					lowercase__=0    ,					lowercase__=1    ,					lowercase__=2    ,					**lowercase__    ,					):






          """simple docstring"""






          super().__init__(**lowercase__    ,					pad_token_id=lowercase__    ,					bos_token_id=lowercase__    ,					eos_token_id=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	hidden_size
          SCREAMING_SNAKE_CASE_				:      List[str]					=	feat_extract_norm
          SCREAMING_SNAKE_CASE_				:      Dict					=	feat_extract_activation
          SCREAMING_SNAKE_CASE_				:      str					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	conv_bias
          SCREAMING_SNAKE_CASE_				:      str					=	num_conv_pos_embeddings
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	num_conv_pos_embedding_groups
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	len(self.conv_dim      )
          SCREAMING_SNAKE_CASE_				:      Any					=	num_hidden_layers
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	intermediate_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	hidden_act
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	num_attention_heads
          SCREAMING_SNAKE_CASE_				:      Any					=	hidden_dropout
          SCREAMING_SNAKE_CASE_				:      List[str]					=	attention_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	activation_dropout
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	feat_proj_layer_norm
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	feat_proj_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	final_dropout
          SCREAMING_SNAKE_CASE_				:      Dict					=	layerdrop
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	layer_norm_eps
          SCREAMING_SNAKE_CASE_				:      Tuple					=	initializer_range
          SCREAMING_SNAKE_CASE_				:      List[str]					=	vocab_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	do_stable_layer_norm
          SCREAMING_SNAKE_CASE_				:      List[str]					=	use_weighted_layer_sum
          SCREAMING_SNAKE_CASE_				:      Any					=	classifier_proj_size

          if (
              (len(self.conv_stride      ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel      ) != self.num_feat_extract_layers)
              or (len(self.conv_dim      ) != self.num_feat_extract_layers)
          ):
                raise ValueError(
                    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                    F" {len(self.conv_dim      )}`, `len(config.conv_stride) = {len(self.conv_stride      )}`,"
                    F" `len(config.conv_kernel) = {len(self.conv_kernel      )}`."      )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          SCREAMING_SNAKE_CASE_				:      Tuple					=	apply_spec_augment
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_time_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_time_length
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	mask_time_min_masks
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	mask_feature_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_feature_length
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_feature_min_masks

          # ctc loss
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_loss_reduction
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_zero_infinity







    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return functools.reduce(operator.mul    ,					self.conv_stride    ,					1      )

