

'''simple docstring'''






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							HerbertTokenizer
    _A         =							HerbertTokenizerFast
    _A         =							True




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"    ,					encoding="utf-8"      ) as f_data:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	f_data.read().replace("\n\n"    ,					"\n"      ).strip()

          SCREAMING_SNAKE_CASE_				:      Dict					=	[
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	dict(zip(lowercase__    ,					range(len(lowercase__      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

          SCREAMING_SNAKE_CASE_				:      Any					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["merges_file"]      )
          with open(self.vocab_file    ,					"w"      ) as fp:
                fp.write(json.dumps(lowercase__      )      )
          with open(self.merges_file    ,					"w"      ) as fp:
                fp.write("\n".join(lowercase__      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	"lower newer"
          SCREAMING_SNAKE_CASE_				:      str					=	"lower newer"
          return input_text, output_text




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.tokenizer_class(vocab_file=self.vocab_file    ,					merges_file=self.merges_file      )

          SCREAMING_SNAKE_CASE_				:      int					=	"lower"
          SCREAMING_SNAKE_CASE_				:      List[str]					=	["low", "er</w>"]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokens + ["<unk>"]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__      )    ,					lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if not self.test_rust_tokenizer:
                return

          SCREAMING_SNAKE_CASE_				:      Dict					=	self.get_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	"lower,newer"

          SCREAMING_SNAKE_CASE_				:      int					=	tokenizer.tokenize(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	rust_tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	rust_tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.encode(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	rust_tokenizer.encode(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )







    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode("konstruowanie sekwencji"    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	tokenizer.encode("konstruowanie wielu sekwencji"    ,					add_special_tokens=lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.build_inputs_with_special_tokens(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.build_inputs_with_special_tokens(lowercase__    ,					lowercase__      )

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]




'''simple docstring'''






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_					  =	logging.get_logger(__name__)

snake_case_					  =	{
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase				):
    _A         =							"hubert"




    def __init__( self    ,					lowercase__=32    ,					lowercase__=768    ,					lowercase__=12    ,					lowercase__=12    ,					lowercase__=3072    ,					lowercase__="gelu"    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=True    ,					lowercase__=0.0    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.02    ,					lowercase__=1e-5    ,					lowercase__="group"    ,					lowercase__="gelu"    ,					lowercase__=(512, 512, 512, 512, 512, 512, 512)    ,					lowercase__=(5, 2, 2, 2, 2, 2, 2)    ,					lowercase__=(10, 3, 3, 3, 3, 2, 2)    ,					lowercase__=False    ,					lowercase__=128    ,					lowercase__=16    ,					lowercase__=False    ,					lowercase__=True    ,					lowercase__=0.05    ,					lowercase__=10    ,					lowercase__=2    ,					lowercase__=0.0    ,					lowercase__=10    ,					lowercase__=0    ,					lowercase__="sum"    ,					lowercase__=False    ,					lowercase__=False    ,					lowercase__=256    ,					lowercase__=0    ,					lowercase__=1    ,					lowercase__=2    ,					**lowercase__    ,					):






          """simple docstring"""






          super().__init__(**lowercase__    ,					pad_token_id=lowercase__    ,					bos_token_id=lowercase__    ,					eos_token_id=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	hidden_size
          SCREAMING_SNAKE_CASE_				:      List[str]					=	feat_extract_norm
          SCREAMING_SNAKE_CASE_				:      Dict					=	feat_extract_activation
          SCREAMING_SNAKE_CASE_				:      str					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	conv_bias
          SCREAMING_SNAKE_CASE_				:      str					=	num_conv_pos_embeddings
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	num_conv_pos_embedding_groups
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	len(self.conv_dim      )
          SCREAMING_SNAKE_CASE_				:      Any					=	num_hidden_layers
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	intermediate_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	hidden_act
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	num_attention_heads
          SCREAMING_SNAKE_CASE_				:      Any					=	hidden_dropout
          SCREAMING_SNAKE_CASE_				:      List[str]					=	attention_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	activation_dropout
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	feat_proj_layer_norm
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	feat_proj_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	final_dropout
          SCREAMING_SNAKE_CASE_				:      Dict					=	layerdrop
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	layer_norm_eps
          SCREAMING_SNAKE_CASE_				:      Tuple					=	initializer_range
          SCREAMING_SNAKE_CASE_				:      List[str]					=	vocab_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	do_stable_layer_norm
          SCREAMING_SNAKE_CASE_				:      List[str]					=	use_weighted_layer_sum
          SCREAMING_SNAKE_CASE_				:      Any					=	classifier_proj_size

          if (
              (len(self.conv_stride      ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel      ) != self.num_feat_extract_layers)
              or (len(self.conv_dim      ) != self.num_feat_extract_layers)
          ):
                raise ValueError(
                    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                    F" {len(self.conv_dim      )}`, `len(config.conv_stride) = {len(self.conv_stride      )}`,"
                    F" `len(config.conv_kernel) = {len(self.conv_kernel      )}`."      )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          SCREAMING_SNAKE_CASE_				:      Tuple					=	apply_spec_augment
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_time_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_time_length
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	mask_time_min_masks
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	mask_feature_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_feature_length
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_feature_min_masks

          # ctc loss
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_loss_reduction
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_zero_infinity







    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return functools.reduce(operator.mul    ,					self.conv_stride    ,					1      )

