



'''simple docstring'''





import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          FunnelBaseModel,
          FunnelForMaskedLM,
          FunnelForMultipleChoice,
          FunnelForPreTraining,
          FunnelForQuestionAnswering,
          FunnelForSequenceClassification,
          FunnelForTokenClassification,
          FunnelModel,
      )





class 							lowerCAmelCase_		:





    def __init__(    self    :    Union[str, Any],    _snake_case    :    List[Any],    _snake_case    :    int=13,    _snake_case    :    int=7,    _snake_case    :    int=True,    _snake_case    :    Dict=True,    _snake_case    :    Optional[int]=True,    _snake_case    :    str=True,    _snake_case    :    int=99,    _snake_case    :    Optional[int]=[1, 1, 2],    _snake_case    :    Dict=1,    _snake_case    :    int=32,    _snake_case    :    List[Any]=4,    _snake_case    :    Any=8,    _snake_case    :    Tuple=37,    _snake_case    :    Tuple="gelu_new",    _snake_case    :    Optional[int]=0.1,    _snake_case    :    Union[str, Any]=0.1,    _snake_case    :    List[str]=0.0,    _snake_case    :    Any=512,    _snake_case    :    str=3,    _snake_case    :    Optional[int]=0.02,    _snake_case    :    Dict=3,    _snake_case    :    Dict=4,    _snake_case    :    List[Any]=None,    _snake_case    :    Tuple=False,    ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =parent
       snake_case   :							int						  =batch_size
       snake_case   :							Tuple						  =seq_length
       snake_case   :							Optional[Any]						  =is_training
       snake_case   :							List[str]						  =use_input_mask
       snake_case   :							Optional[Any]						  =use_token_type_ids
       snake_case   :							str						  =use_labels
       snake_case   :							List[str]						  =vocab_size
       snake_case   :							List[str]						  =block_sizes
       snake_case   :							Any						  =num_decoder_layers
       snake_case   :							Optional[int]						  =d_model
       snake_case   :							Union[str, Any]						  =n_head
       snake_case   :							Optional[Any]						  =d_head
       snake_case   :							Union[str, Any]						  =d_inner
       snake_case   :							Optional[Any]						  =hidden_act
       snake_case   :							Optional[int]						  =hidden_dropout
       snake_case   :							int						  =attention_dropout
       snake_case   :							Optional[Any]						  =activation_dropout
       snake_case   :							Union[str, Any]						  =max_position_embeddings
       snake_case   :							List[Any]						  =type_vocab_size
       snake_case   :							Tuple						  =2
       snake_case   :							Optional[int]						  =num_labels
       snake_case   :							Any						  =num_choices
       snake_case   :							Tuple						  =scope
       snake_case   :							Union[str, Any]						  =initializer_std

       # Used in the tests to check the size of the first attention layer
       snake_case   :							int						  =n_head
       # Used in the tests to check the size of the first hidden state
       snake_case   :							Optional[Any]						  =self.d_model
       # Used in the tests to check the number of output hidden states/attentions
       snake_case   :							Tuple						  =sum(self.block_sizes     ) + (0 if base else self.num_decoder_layers)
       # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
       # the last hidden state of the first block (which is the first hidden state of the decoder).
       if not base:
          snake_case   :							Tuple						  =self.num_hidden_layers + 2





    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =ids_tensor([self.batch_size, self.seq_length],    self.vocab_size     )

       snake_case   :							Dict						  =None
       if self.use_input_mask:
          snake_case   :							Dict						  =ids_tensor([self.batch_size, self.seq_length],    vocab_size=2     )

       snake_case   :							List[Any]						  =None
       if self.use_token_type_ids:
          snake_case   :							List[str]						  =ids_tensor([self.batch_size, self.seq_length],    self.type_vocab_size     )

       snake_case   :							List[str]						  =None
       snake_case   :							Optional[Any]						  =None
       snake_case   :							Tuple						  =None
       if self.use_labels:
          snake_case   :							Tuple						  =ids_tensor([self.batch_size],    self.type_sequence_label_size     )
          snake_case   :							str						  =ids_tensor([self.batch_size, self.seq_length],    self.num_labels     )
          snake_case   :							Dict						  =ids_tensor([self.batch_size],    self.num_choices     )
          snake_case   :							Tuple						  =ids_tensor([self.batch_size, self.seq_length],    1     )

       snake_case   :							str						  =self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           fake_token_labels,
       )





    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       return FunnelConfig(
           vocab_size=self.vocab_size,    block_sizes=self.block_sizes,    num_decoder_layers=self.num_decoder_layers,    d_model=self.d_model,    n_head=self.n_head,    d_head=self.d_head,    d_inner=self.d_inner,    hidden_act=self.hidden_act,    hidden_dropout=self.hidden_dropout,    attention_dropout=self.attention_dropout,    activation_dropout=self.activation_dropout,    max_position_embeddings=self.max_position_embeddings,    type_vocab_size=self.type_vocab_size,    initializer_std=self.initializer_std,    )





    def 					__snake_case       (    self    :    Dict,    _snake_case    :    str,    _snake_case    :    Optional[Any],    _snake_case    :    Any,    _snake_case    :    Tuple,    _snake_case    :    Union[str, Any],    _snake_case    :    Union[str, Any],    _snake_case    :    Any,    _snake_case    :    int,    ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =FunnelModel(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Any						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case     )
       snake_case   :							Optional[int]						  =model(_snake_case,    token_type_ids=_snake_case     )
       snake_case   :							Any						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )

       snake_case   :							int						  =False
       snake_case   :							Any						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )

       snake_case   :							Dict						  =False
       snake_case   :							List[str]						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.d_model)     )





    def 					__snake_case       (    self    :    List[str],    _snake_case    :    Dict,    _snake_case    :    Any,    _snake_case    :    Dict,    _snake_case    :    List[str],    _snake_case    :    Dict,    _snake_case    :    List[str],    _snake_case    :    Optional[int],    _snake_case    :    Union[str, Any],    ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =FunnelBaseModel(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							str						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case     )
       snake_case   :							Dict						  =model(_snake_case,    token_type_ids=_snake_case     )
       snake_case   :							str						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 2, self.d_model)     )

       snake_case   :							str						  =False
       snake_case   :							Optional[int]						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 3, self.d_model)     )

       snake_case   :							Union[str, Any]						  =False
       snake_case   :							Tuple						  =model(_snake_case     )
       self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, 2, self.d_model)     )





    def 					__snake_case       (    self    :    int,    _snake_case    :    str,    _snake_case    :    Tuple,    _snake_case    :    str,    _snake_case    :    Tuple,    _snake_case    :    str,    _snake_case    :    Dict,    _snake_case    :    str,    _snake_case    :    Dict,    ):



       '''simple docstring'''







       snake_case   :							Dict						  =self.num_labels
       snake_case   :							Optional[int]						  =FunnelForPreTraining(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Any						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length)     )





    def 					__snake_case       (    self    :    Tuple,    _snake_case    :    List[Any],    _snake_case    :    Tuple,    _snake_case    :    int,    _snake_case    :    Optional[Any],    _snake_case    :    int,    _snake_case    :    int,    _snake_case    :    Optional[Any],    _snake_case    :    List[Any],    ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =FunnelForMaskedLM(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Union[str, Any]						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.vocab_size)     )





    def 					__snake_case       (    self    :    int,    _snake_case    :    int,    _snake_case    :    str,    _snake_case    :    List[str],    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    _snake_case    :    Optional[Any],    _snake_case    :    Optional[int],    _snake_case    :    Any,    ):



       '''simple docstring'''







       snake_case   :							Dict						  =self.num_labels
       snake_case   :							Optional[int]						  =FunnelForSequenceClassification(_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Optional[Any]						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.num_labels)     )





    def 					__snake_case       (    self    :    Tuple,    _snake_case    :    Union[str, Any],    _snake_case    :    List[str],    _snake_case    :    int,    _snake_case    :    Tuple,    _snake_case    :    List[str],    _snake_case    :    List[str],    _snake_case    :    int,    _snake_case    :    Any,    ):



       '''simple docstring'''







       snake_case   :							Tuple						  =self.num_choices
       snake_case   :							Union[str, Any]						  =FunnelForMultipleChoice(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							str						  =input_ids.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
       snake_case   :							str						  =token_type_ids.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
       snake_case   :							Tuple						  =input_mask.unsqueeze(1     ).expand(-1,    self.num_choices,    -1     ).contiguous()
       snake_case   :							str						  =model(
           _snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    labels=_snake_case,    )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.num_choices)     )





    def 					__snake_case       (    self    :    str,    _snake_case    :    int,    _snake_case    :    List[str],    _snake_case    :    str,    _snake_case    :    Optional[Any],    _snake_case    :    int,    _snake_case    :    str,    _snake_case    :    str,    _snake_case    :    int,    ):



       '''simple docstring'''







       snake_case   :							Any						  =self.num_labels
       snake_case   :							List[Any]						  =FunnelForTokenClassification(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Dict						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    labels=_snake_case     )
       self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.num_labels)     )





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    List[str],    _snake_case    :    List[str],    _snake_case    :    List[str],    _snake_case    :    List[str],    _snake_case    :    Optional[Any],    _snake_case    :    Union[str, Any],    _snake_case    :    Union[str, Any],    _snake_case    :    Tuple,    ):



       '''simple docstring'''







       snake_case   :							Tuple						  =FunnelForQuestionAnswering(config=_snake_case     )
       model.to(_snake_case     )
       model.eval()
       snake_case   :							Tuple						  =model(
           _snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case,    start_positions=_snake_case,    end_positions=_snake_case,    )
       self.parent.assertEqual(result.start_logits.shape,    (self.batch_size, self.seq_length)     )
       self.parent.assertEqual(result.end_logits.shape,    (self.batch_size, self.seq_length)     )






    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.prepare_config_and_inputs()
       (
           (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    (
           snake_case
       )  ,    
       )   :							Dict						  =config_and_inputs
       snake_case   :							str						  ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict





@require_torch
class 							lowerCAmelCase_		(	a_  ,      a_  ,      unittest.TestCase						):
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	(
        (
            FunnelModel,
            FunnelForMaskedLM,
            FunnelForPreTraining,
            FunnelForQuestionAnswering,
            FunnelForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    __UpperCAmelCase							=	(
        {
            'feature-extraction': (FunnelBaseModel, FunnelModel),
            'fill-mask': FunnelForMaskedLM,
            'question-answering': FunnelForQuestionAnswering,
            'text-classification': FunnelForSequenceClassification,
            'token-classification': FunnelForTokenClassification,
            'zero-shot': FunnelForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )





    def 					__snake_case       (    self    :    int,    _snake_case    :    str,    _snake_case    :    Any,    _snake_case    :    Union[str, Any]=False     ):



       '''simple docstring'''







       snake_case   :							Dict						  =super()._prepare_for_class(_snake_case,    _snake_case,    return_labels=_snake_case     )

       if return_labels:
          if model_class in get_values(_snake_case     ):
             snake_case   :							List[Any]						  =torch.zeros(
                 (self.model_tester.batch_size, self.model_tester.seq_length),    dtype=torch.long,    device=_snake_case     )
       return inputs_dict





    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =FunnelModelTester(self     )
       snake_case   :							List[Any]						  =ConfigTester(self,    config_class=_snake_case     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       self.config_tester.run_common_tests()





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_snake_case     )





    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*_snake_case     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Union[str, Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_snake_case     )





    def 					__snake_case       (    self    :    Dict     ):



       '''simple docstring'''







       snake_case   :							Optional[Any]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_snake_case     )





    def 					__snake_case       (    self    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_snake_case     )






    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       if hasattr(_snake_case,    '''weight'''     ) and module.weight is not None:
          module.weight.data.fill_(3     )
       if hasattr(_snake_case,    '''bias'''     ) and module.bias is not None:
          module.bias.data.fill_(3     )

       for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
          if hasattr(_snake_case,    _snake_case     ) and getattr(_snake_case,    _snake_case     ) is not None:
             snake_case   :							str						  =getattr(_snake_case,    _snake_case     )
             weight.data.fill_(3     )





@require_torch
class 							lowerCAmelCase_		(	a_  ,      unittest.TestCase						):
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	False
    __UpperCAmelCase							=	(
        (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
    )





    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       snake_case   :							Tuple						  =FunnelModelTester(self,    base=_snake_case     )
       snake_case   :							Tuple						  =ConfigTester(self,    config_class=_snake_case     )





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       self.config_tester.run_common_tests()





    def 					__snake_case       (    self    :    Optional[Any]     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_base_model(*_snake_case     )





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							Any						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_snake_case     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							List[str]						  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*_snake_case     )





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case  ,    snake_case   :							Tuple						  =self.model_tester.prepare_config_and_inputs_for_common()
       snake_case   :							Dict						  =True

       for model_class in self.all_model_classes:
          if model_class.__name__ == "FunnelBaseModel":
             continue
          snake_case   :							str						  =model_class(_snake_case     )
          model.to(_snake_case     )
          model.train()
          snake_case   :							Union[str, Any]						  =self._prepare_for_class(_snake_case,    _snake_case,    return_labels=_snake_case     )
          snake_case   :							Dict						  =model(**_snake_case     ).loss
          loss.backward()






    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    Optional[int]     ):



       '''simple docstring'''







       if hasattr(_snake_case,    '''weight'''     ) and module.weight is not None:
          module.weight.data.fill_(3     )
       if hasattr(_snake_case,    '''bias'''     ) and module.bias is not None:
          module.bias.data.fill_(3     )

       for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
          if hasattr(_snake_case,    _snake_case     ) and getattr(_snake_case,    _snake_case     ) is not None:
             snake_case   :							Optional[int]						  =getattr(_snake_case,    _snake_case     )
             weight.data.fill_(3     )





@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase_		(	unittest.TestCase						):





    def 					__snake_case       (    self    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							int						  =13
       snake_case   :							Union[str, Any]						  =7
       snake_case   :							Optional[int]						  =torch.arange(0,    batch_size * sequence_length     ).long().reshape(_snake_case,    _snake_case     )
       snake_case   :							Optional[int]						  =[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
       snake_case   :							Any						  =torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]     )

       snake_case   :							List[Any]						  =FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''     )
       snake_case   :							Dict						  =model(_snake_case,    token_type_ids=_snake_case     )[0].abs()

       snake_case   :							str						  =torch.tensor(2344.8352     )
       snake_case   :							Any						  =torch.tensor(0.8052     )
       self.assertTrue(torch.allclose(output.sum(),    _snake_case,    atol=1E-4     )     )
       self.assertTrue(torch.allclose(output.mean(),    _snake_case,    atol=1E-4     )     )

       snake_case   :							Tuple						  =torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]     )
       snake_case   :							str						  =model(_snake_case,    attention_mask=_snake_case,    token_type_ids=_snake_case     )[0].abs()

       snake_case   :							Optional[Any]						  =torch.tensor(2343.8425     )
       snake_case   :							int						  =torch.tensor(0.8049     )
       self.assertTrue(torch.allclose(output.sum(),    _snake_case,    atol=1E-4     )     )
       self.assertTrue(torch.allclose(output.mean(),    _snake_case,    atol=1E-4     )     )






    @slow
    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       snake_case   :							Dict						  =FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''     )
       snake_case   :							Any						  =FunnelModel.from_pretrained('''huggingface/funnel-small'''     )
       snake_case   :							Dict						  =tokenizer('''Hello! I am the Funnel Transformer model.''',    return_tensors='''pt'''     )
       snake_case   :							Optional[Any]						  =model(**_snake_case     )[0]

       snake_case   :							Dict						  =torch.tensor(235.7246     )
       snake_case   :							int						  =torch.tensor(0.0256     )
       self.assertTrue(torch.allclose(output.sum(),    _snake_case,    atol=1E-4     )     )
       self.assertTrue(torch.allclose(output.mean(),    _snake_case,    atol=1E-4     )     )










'''simple docstring'''





import argparse

import torch
from saprl.hub import distilhubert

from transformers import HubertConfig, HubertModel, WavaVecaFeatureExtractor, logging


logging.set_verbosity_info()
A			:   Dict			=      logging.get_logger(__name__)

A			:   Tuple			=      {
    """post_extract_proj""": """feature_projection.projection""",
    """encoder.pos_conv.0""": """encoder.pos_conv_embed.conv""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """mask_emb""": """masked_spec_embed""",
}
def 		_a   (  lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_							):
   for attribute in key.split('''.'''							):
      snake_case   :							Optional[Any]						  =getattr(lowerCamelCase_					,						lowerCamelCase_							)

   if weight_type is not None:
      snake_case   :							int						  =getattr(lowerCamelCase_					,						lowerCamelCase_							).shape
   else:
      snake_case   :							Any						  =hf_pointer.shape

   assert hf_shape == value.shape, (
       F'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
       F''' {value.shape} for {full_name}'''
   )

   if weight_type == "weight":
      snake_case   :							Dict						  =value
   elif weight_type == "weight_g":
      snake_case   :							Any						  =value
   elif weight_type == "weight_v":
      snake_case   :							int						  =value
   elif weight_type == "bias":
      snake_case   :							int						  =value
   else:
      snake_case   :							str						  =value

   logger.info(F'''{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'''							)
def 		_a   (  lowerCamelCase_					,						lowerCamelCase_							):
   snake_case   :							str						  =[]
   snake_case   :							Tuple						  =fairseq_model.state_dict()

   snake_case   :							Union[str, Any]						  =hf_model.feature_extractor

   for name, value in fairseq_dict.items():
      snake_case   :							Optional[int]						  =False
      if "conv_layers" in name:
         load_conv_layer(
             lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						hf_model.config.feat_extract_norm == '''group'''					,						)
         snake_case   :							int						  =True
      else:
         for key, mapped_key in MAPPING.items():
            snake_case   :							Dict						  =mapped_key

            if key in name:
               snake_case   :							str						  =True
               if "*" in mapped_key:
                  snake_case   :							Tuple						  =name.split(lowerCamelCase_							)[0].split('''.'''							)[-2]
                  snake_case   :							Optional[int]						  =mapped_key.replace('''*'''					,						lowerCamelCase_							)
               if "weight_g" in name:
                  snake_case   :							int						  ='''weight_g'''
               elif "weight_v" in name:
                  snake_case   :							List[str]						  ='''weight_v'''
               elif "weight" in name:
                  snake_case   :							Any						  ='''weight'''
               elif "bias" in name:
                  snake_case   :							List[str]						  ='''bias'''
               else:
                  snake_case   :							Tuple						  =None
               set_recursively(lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_							)
            continue
      if not is_used:
         unused_weights.append(lowerCamelCase_							)

   logger.warning(F'''Unused weights: {unused_weights}'''							)
def 		_a   (  lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_					,						lowerCamelCase_							):
   snake_case   :							List[str]						  =full_name.split('''conv_layers.'''							)[-1]
   snake_case   :							Any						  =name.split('''.'''							)
   snake_case   :							Tuple						  =int(items[0]							)
   snake_case   :							int						  =int(items[1]							)

   if type_id == 0:
      if "bias" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
             F'''{full_name} has size {value.shape}, but'''
             F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
         )
         snake_case   :							Optional[int]						  =value
         logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''							)
      elif "weight" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
             F'''{full_name} has size {value.shape}, but'''
             F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
         )
         snake_case   :							str						  =value
         logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''							)
   elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
      if "bias" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
             F'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
             " found."
         )
         snake_case   :							Optional[int]						  =value
         logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''							)
      elif "weight" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
             F'''{full_name} has size {value.shape}, but'''
             F''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
         )
         snake_case   :							Union[str, Any]						  =value
         logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''							)
   else:
      unused_weights.append(lowerCamelCase_							)
def 		_a   (  lowerCamelCase_							):
   snake_case   :							Any						  =HubertConfig()
   snake_case   :							List[str]						  =model.config

   snake_case   :							List[str]						  =fs_config.activation_dropout
   snake_case   :							List[Any]						  =False
   snake_case   :							Optional[Any]						  =fs_config.attention_dropout
   snake_case   :							Optional[int]						  =False
   snake_case   :							Any						  =eval(fs_config.extractor_conv_feature_layers							)
   snake_case   :							Optional[int]						  =[x[0] for x in conv_layers]
   snake_case   :							Any						  =[x[1] for x in conv_layers]
   snake_case   :							Optional[Any]						  =[x[2] for x in conv_layers]
   snake_case   :							Optional[Any]						  ='''gelu'''
   snake_case   :							Optional[Any]						  ='''layer''' if fs_config.extractor_mode == '''layer_norm''' else '''group'''
   snake_case   :							Optional[int]						  =False
   snake_case   :							Optional[Any]						  =0.0
   snake_case   :							Optional[int]						  =0.0
   snake_case   :							Dict						  =fs_config.activation_fn
   snake_case   :							Any						  =fs_config.dropout
   snake_case   :							Optional[int]						  =fs_config.encoder_embed_dim
   snake_case   :							Dict						  =0.02
   snake_case   :							Dict						  =fs_config.encoder_ffn_embed_dim
   snake_case   :							str						  =1e-5
   snake_case   :							List[Any]						  =0.0
   snake_case   :							Dict						  =fs_config.encoder_attention_heads
   snake_case   :							Union[str, Any]						  =fs_config.conv_pos_groups
   snake_case   :							Union[str, Any]						  =fs_config.conv_pos
   snake_case   :							Dict						  =len(lowerCamelCase_							)
   snake_case   :							str						  =fs_config.encoder_layers

   return config
@torch.no_grad()
def 		_a   (  lowerCamelCase_					,						lowerCamelCase_=None							):
   snake_case   :							Union[str, Any]						  =distilhubert().model.model

   if config_path is not None:
      snake_case   :							int						  =HubertConfig.from_pretrained(lowerCamelCase_							)
   else:
      snake_case   :							Optional[Any]						  =convert_config(lowerCamelCase_							)
   snake_case   :							Tuple						  =model.eval()

   snake_case   :							Optional[Any]						  =WavaVecaFeatureExtractor(
       feature_size=1					,						sampling_rate=1_60_00					,						padding_value=0					,						do_normalize=lowerCamelCase_					,						return_attention_mask=lowerCamelCase_					,						)
   snake_case   :							Any						  =HubertModel(lowerCamelCase_							)

   recursively_load_weights(lowerCamelCase_					,						lowerCamelCase_							)

   feature_extractor.save_pretrained(lowerCamelCase_							)
   hf_model.save_pretrained(lowerCamelCase_							)


if __name__ == "__main__":
      A			:   Tuple			=      argparse.ArgumentParser()
      parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
      parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
      A			:   int			=      parser.parse_args()
      convert_hubert_checkpoint(args.pytorch_dump_folder_path, args.config_path)





