

'''simple docstring'''






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece
class      SCREAMING_SNAKE_CASE__      (    unittest.TestCase				):




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Tuple					=	list(tokenizer_class.max_model_input_sizes.keys()      )
          for model_name in sa_models[:1]:
                SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer_class.from_pretrained(lowercase__      )
                self.assertIsNotNone(lowercase__      )
                self.assertIsInstance(lowercase__    ,					lowercase__      )
                self.assertIsInstance(lowercase__    ,					lowercase__      )

                for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(lowercase__    ,					lowercase__      )
                      SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.convert_tokens_to_ids(lowercase__      )
                      self.assertIsInstance(lowercase__    ,					lowercase__      )




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	pickle.dumps(lowercase__      )
          self.assertIsNotNone(lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	pickle.loads(lowercase__      )

          # Ensure is_fast is correctly restored
          self.assertEqual(be_restored.is_fast    ,					be_original.is_fast      )

          # Ensure encodings are potentially correctly restored
          if be_original.is_fast:
                self.assertIsNotNone(be_restored.encodings      )
          else:
                self.assertIsNone(be_restored.encodings      )

          # Ensure the keys are the same
          for original_v, restored_v in zip(be_original.values()    ,					be_restored.values()      ):
                if equal_op:
                      self.assertTrue(equal_op(lowercase__    ,					lowercase__      )      )
                else:
                      self.assertEqual(lowercase__    ,					lowercase__      )




    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          self.check_tokenizer_from_pretrained(lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          self.assertEqual(TensorType("tf"      )    ,					TensorType.TENSORFLOW      )
          self.assertEqual(TensorType("pt"      )    ,					TensorType.PYTORCH      )
          self.assertEqual(TensorType("np"      )    ,					TensorType.NUMPY      )




    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          import numpy as np

          SCREAMING_SNAKE_CASE_				:      Any					=	BertTokenizer.from_pretrained("bert-base-cased"      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	BertTokenizerFast.from_pretrained("bert-base-cased"      )

          # Python no tensor
          with self.subTest("BatchEncoding (Python, return_tensors=None)"      ):
                self.assert_dump_and_restore(tokenizer_p("Small example to encode"      )      )

          with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"      ):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"    ,					return_tensors=TensorType.NUMPY      )    ,					np.array_equal      )

          with self.subTest("BatchEncoding (Rust, return_tensors=None)"      ):
                self.assert_dump_and_restore(tokenizer_r("Small example to encode"      )      )

          with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"      ):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"    ,					return_tensors=TensorType.NUMPY      )    ,					np.array_equal      )




    @require_tf
    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          import tensorflow as tf

          def tf_array_equals(lowercase__    ,					lowercase__      ):
                return tf.reduce_all(tf.equal(lowercase__    ,					lowercase__      )      )

          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	BertTokenizer.from_pretrained("bert-base-cased"      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	BertTokenizerFast.from_pretrained("bert-base-cased"      )

          with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"      ):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"    ,					return_tensors=TensorType.TENSORFLOW      )    ,					lowercase__      )

          with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"      ):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"    ,					return_tensors=TensorType.TENSORFLOW      )    ,					lowercase__      )




    @require_torch
    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          import torch

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	BertTokenizer.from_pretrained("bert-base-cased"      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	BertTokenizerFast.from_pretrained("bert-base-cased"      )

          with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"      ):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"    ,					return_tensors=TensorType.PYTORCH      )    ,					torch.equal      )

          with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"      ):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"    ,					return_tensors=TensorType.PYTORCH      )    ,					torch.equal      )




    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      List[Any]					=	BertTokenizer.from_pretrained("bert-base-cased"      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	BertTokenizerFast.from_pretrained("bert-base-cased"      )

          with self.subTest("Python Tokenizer"      ):
                self.assertFalse(tokenizer_p("Small example to_encode"      ).is_fast      )

          with self.subTest("Rust Tokenizer"      ):
                self.assertTrue(tokenizer_r("Small example to_encode"      ).is_fast      )




    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Any					=	BertTokenizerFast.from_pretrained("bert-base-cased"      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer_r(["Test", "\xad", "test"]    ,					is_split_into_words=lowercase__      )

          self.assertEqual(encoded.word_to_tokens(0      )    ,					TokenSpan(start=1    ,					end=2      )      )
          self.assertEqual(encoded.word_to_tokens(1      )    ,					lowercase__      )
          self.assertEqual(encoded.word_to_tokens(2      )    ,					TokenSpan(start=2    ,					end=3      )      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}      )
          SCREAMING_SNAKE_CASE_				:      Tuple					=	batch.convert_to_tensors(tensor_type="np"      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(2, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(2,)      )
          # test converting the converted
          with CaptureStderr() as cs:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	batch.convert_to_tensors(tensor_type="np"      )
          self.assertFalse(len(cs.err      )    ,					msg=F"should have no warning, but got {cs.err}"      )

          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0}      )
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	batch.convert_to_tensors(tensor_type="np"    ,					prepend_batch_axis=lowercase__      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(1, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(1,)      )




    @require_torch
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      List[str]					=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}      )
          SCREAMING_SNAKE_CASE_				:      str					=	batch.convert_to_tensors(tensor_type="pt"      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(2, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(2,)      )
          # test converting the converted
          with CaptureStderr() as cs:
                SCREAMING_SNAKE_CASE_				:      Optional[int]					=	batch.convert_to_tensors(tensor_type="pt"      )
          self.assertFalse(len(cs.err      )    ,					msg=F"should have no warning, but got {cs.err}"      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0}      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	batch.convert_to_tensors(tensor_type="pt"    ,					prepend_batch_axis=lowercase__      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(1, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(1,)      )




    @require_tf
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}      )
          SCREAMING_SNAKE_CASE_				:      str					=	batch.convert_to_tensors(tensor_type="tf"      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(2, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(2,)      )
          # test converting the converted
          with CaptureStderr() as cs:
                SCREAMING_SNAKE_CASE_				:      int					=	batch.convert_to_tensors(tensor_type="tf"      )
          self.assertFalse(len(cs.err      )    ,					msg=F"should have no warning, but got {cs.err}"      )

          SCREAMING_SNAKE_CASE_				:      Any					=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0}      )
          SCREAMING_SNAKE_CASE_				:      Tuple					=	batch.convert_to_tensors(tensor_type="tf"    ,					prepend_batch_axis=lowercase__      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(1, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(1,)      )




    @require_flax
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}      )
          SCREAMING_SNAKE_CASE_				:      Any					=	batch.convert_to_tensors(tensor_type="jax"      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(2, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(2,)      )
          # test converting the converted
          with CaptureStderr() as cs:
                SCREAMING_SNAKE_CASE_				:      Optional[int]					=	batch.convert_to_tensors(tensor_type="jax"      )
          self.assertFalse(len(cs.err      )    ,					msg=F"should have no warning, but got {cs.err}"      )

          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	BatchEncoding({"inputs": [1, 2, 3], "labels": 0}      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	batch.convert_to_tensors(tensor_type="jax"    ,					prepend_batch_axis=lowercase__      )
          self.assertEqual(tensor_batch["inputs"].shape    ,					(1, 3)      )
          self.assertEqual(tensor_batch["labels"].shape    ,					(1,)      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	[{"input_ids": np.array([0, 1, 2]      )}, {"input_ids": np.array([0, 1, 2, 3]      )}]
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	BertTokenizer.from_pretrained("bert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.pad(lowercase__    ,					padding=lowercase__      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					np.ndarray      )      )
          self.assertEqual(batch["input_ids"].tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.pad(lowercase__    ,					padding=lowercase__    ,					return_tensors="np"      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					np.ndarray      )      )
          self.assertEqual(batch["input_ids"].tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )




    @require_torch
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          import torch

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	[{"input_ids": torch.tensor([0, 1, 2]      )}, {"input_ids": torch.tensor([0, 1, 2, 3]      )}]
          SCREAMING_SNAKE_CASE_				:      int					=	BertTokenizer.from_pretrained("bert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      Tuple					=	tokenizer.pad(lowercase__    ,					padding=lowercase__      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					torch.Tensor      )      )
          self.assertEqual(batch["input_ids"].tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.pad(lowercase__    ,					padding=lowercase__    ,					return_tensors="pt"      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					torch.Tensor      )      )
          self.assertEqual(batch["input_ids"].tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )




    @require_tf
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          import tensorflow as tf

          SCREAMING_SNAKE_CASE_				:      str					=	[{"input_ids": tf.constant([0, 1, 2]      )}, {"input_ids": tf.constant([0, 1, 2, 3]      )}]
          SCREAMING_SNAKE_CASE_				:      str					=	BertTokenizer.from_pretrained("bert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.pad(lowercase__    ,					padding=lowercase__      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					tf.Tensor      )      )
          self.assertEqual(batch["input_ids"].numpy().tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.pad(lowercase__    ,					padding=lowercase__    ,					return_tensors="tf"      )
          self.assertTrue(isinstance(batch["input_ids"]    ,					tf.Tensor      )      )
          self.assertEqual(batch["input_ids"].numpy().tolist()    ,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )




    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	Tokenizer(WordPiece(unk_token="[UNK]"      )      )
          PreTrainedTokenizerFast(tokenizer_object=lowercase__      )







    @require_tokenizers
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Tuple					=	Tokenizer(WordPiece(unk_token="[UNK]"      )      )
          with tempfile.TemporaryDirectory() as tmpdirname:
                bert_tokenizer.save(os.path.join(lowercase__    ,					"tokenizer.json"      )      )
                PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowercase__    ,					"tokenizer.json"      )      )




'''simple docstring'''






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_					  =	logging.get_logger(__name__)

snake_case_					  =	{
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase				):
    _A         =							"hubert"




    def __init__( self    ,					lowercase__=32    ,					lowercase__=768    ,					lowercase__=12    ,					lowercase__=12    ,					lowercase__=3072    ,					lowercase__="gelu"    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=True    ,					lowercase__=0.0    ,					lowercase__=0.1    ,					lowercase__=0.1    ,					lowercase__=0.02    ,					lowercase__=1e-5    ,					lowercase__="group"    ,					lowercase__="gelu"    ,					lowercase__=(512, 512, 512, 512, 512, 512, 512)    ,					lowercase__=(5, 2, 2, 2, 2, 2, 2)    ,					lowercase__=(10, 3, 3, 3, 3, 2, 2)    ,					lowercase__=False    ,					lowercase__=128    ,					lowercase__=16    ,					lowercase__=False    ,					lowercase__=True    ,					lowercase__=0.05    ,					lowercase__=10    ,					lowercase__=2    ,					lowercase__=0.0    ,					lowercase__=10    ,					lowercase__=0    ,					lowercase__="sum"    ,					lowercase__=False    ,					lowercase__=False    ,					lowercase__=256    ,					lowercase__=0    ,					lowercase__=1    ,					lowercase__=2    ,					**lowercase__    ,					):






          """simple docstring"""






          super().__init__(**lowercase__    ,					pad_token_id=lowercase__    ,					bos_token_id=lowercase__    ,					eos_token_id=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	hidden_size
          SCREAMING_SNAKE_CASE_				:      List[str]					=	feat_extract_norm
          SCREAMING_SNAKE_CASE_				:      Dict					=	feat_extract_activation
          SCREAMING_SNAKE_CASE_				:      str					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	list(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	conv_bias
          SCREAMING_SNAKE_CASE_				:      str					=	num_conv_pos_embeddings
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	num_conv_pos_embedding_groups
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	len(self.conv_dim      )
          SCREAMING_SNAKE_CASE_				:      Any					=	num_hidden_layers
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	intermediate_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	hidden_act
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	num_attention_heads
          SCREAMING_SNAKE_CASE_				:      Any					=	hidden_dropout
          SCREAMING_SNAKE_CASE_				:      List[str]					=	attention_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	activation_dropout
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	feat_proj_layer_norm
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	feat_proj_dropout
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	final_dropout
          SCREAMING_SNAKE_CASE_				:      Dict					=	layerdrop
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	layer_norm_eps
          SCREAMING_SNAKE_CASE_				:      Tuple					=	initializer_range
          SCREAMING_SNAKE_CASE_				:      List[str]					=	vocab_size
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	do_stable_layer_norm
          SCREAMING_SNAKE_CASE_				:      List[str]					=	use_weighted_layer_sum
          SCREAMING_SNAKE_CASE_				:      Any					=	classifier_proj_size

          if (
              (len(self.conv_stride      ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel      ) != self.num_feat_extract_layers)
              or (len(self.conv_dim      ) != self.num_feat_extract_layers)
          ):
                raise ValueError(
                    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                    F" {len(self.conv_dim      )}`, `len(config.conv_stride) = {len(self.conv_stride      )}`,"
                    F" `len(config.conv_kernel) = {len(self.conv_kernel      )}`."      )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          SCREAMING_SNAKE_CASE_				:      Tuple					=	apply_spec_augment
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_time_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_time_length
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	mask_time_min_masks
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	mask_feature_prob
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	mask_feature_length
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	mask_feature_min_masks

          # ctc loss
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_loss_reduction
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	ctc_zero_infinity







    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return functools.reduce(operator.mul    ,					self.conv_stride    ,					1      )

