






import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase
@require_tokenizers
class    a   (			unittest.TestCase       ):



      """simple docstring"""


      a  :			str								=     EsmTokenizer
      def      UpperCAmelCase				(				self :      Optional[int] )							->				Tuple:
       super().setUp()
       __UpperCAmelCase							:     Tuple									=  tempfile.mkdtemp()
       # fmt: off
       __UpperCAmelCase							:     int									=  ["""<cls>""", """<pad>""", """<eos>""", """<unk>""", """L""", """A""", """G""", """V""", """S""", """E""", """R""", """T""", """I""", """D""", """P""", """K""", """Q""", """N""", """F""", """Y""", """M""", """H""", """W""", """C""", """X""", """B""", """U""", """Z""", """O""", """.""", """-""", """<null_1>""", """<mask>"""]  # noqa: E501
       # fmt: on
       __UpperCAmelCase							:     List[str]									=  os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["""vocab_file"""] )
       with open(self.vocab_file		,				"""w"""		,				encoding="""utf-8""" ) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )
      def      UpperCAmelCase				(				self :      Dict		,				**__lowercase :      Optional[int] )							->				Dict:
       return [self.get_tokenizer(**__UpperCamelCase )]
      def      UpperCAmelCase				(				self :      List[Any]		,				**__lowercase :      List[str] )							->				List[str]:
       return self.tokenizer_class.from_pretrained(self.tmpdirname		,				**__UpperCamelCase )
      def      UpperCAmelCase				(				self :      Dict )							->				Optional[int]:
       __UpperCAmelCase							:     Optional[Any]									=  self.tokenizer_class(self.vocab_file )

       __UpperCAmelCase							:     Optional[int]									=  tokenizer.tokenize("""LAGVS""" )
       self.assertListEqual(__UpperCamelCase		,				["""L""", """A""", """G""", """V""", """S"""] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase )		,				[4, 5, 6, 7, 8] )
      def      UpperCAmelCase				(				self :      int )							->				Tuple:
       __UpperCAmelCase							:     Optional[Any]									=  self.tokenizer_class(self.vocab_file )

       __UpperCAmelCase							:     str									=  """LAGVS"""
       self.assertListEqual(tokenizer.encode(__UpperCamelCase )		,				[0, 4, 5, 6, 7, 8, 2] )
      def      UpperCAmelCase				(				self :      Optional[int] )							->				str:
       __UpperCAmelCase							:     List[Any]									=  self.tokenizer_class(self.vocab_file )

       __UpperCAmelCase							:     Any									=  ["""LAGVS""", """WCB"""]
       __UpperCAmelCase							:     Any									=  tokenizer(__UpperCamelCase		,				padding=__UpperCamelCase )["""input_ids"""]

       self.assertListEqual(__UpperCamelCase		,				[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]] )
      def      UpperCAmelCase				(				self :      Optional[int] )							->				Tuple:
       __UpperCAmelCase							:     int									=  self.tokenizer_class(self.vocab_file )

       __UpperCAmelCase							:     Tuple									=  ["""LAGVS""", """WCB"""]
       __UpperCAmelCase							:     Tuple									=  tokenizer(__UpperCamelCase		,				padding=__UpperCamelCase )["""input_ids"""]

       self.assertListEqual(__UpperCamelCase		,				[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]] )




      def      UpperCAmelCase				(				self :      int )							->				Optional[int]:
       __UpperCAmelCase							:     Any									=  self.get_tokenizers(fast=__UpperCamelCase )
       for tokenizer in tokenizers:
        with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
         __UpperCAmelCase							:     Tuple									=  """<unk>"""
         __UpperCAmelCase							:     Optional[int]									=  """<mask>"""

         __UpperCAmelCase							:     Tuple									=  tokenizer.tokenize(__UpperCamelCase )
         __UpperCAmelCase							:     Tuple									=  tokenizer.tokenize(__UpperCamelCase )

         self.assertEqual(len(__UpperCamelCase )		,				1 )
         self.assertEqual(len(__UpperCamelCase )		,				1 )
         self.assertEqual(token_a[0]		,				__UpperCamelCase )
         self.assertEqual(token_a[0]		,				__UpperCamelCase )










import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


a		:	Optional[Any]      					=			logging.get_logger(__name__)

a		:	int      					=			{
    "facebook/musicgen-small": "https://huggingface.co/facebook/musicgen-small/resolve/main/config.json",
    # See all Musicgen models at https://huggingface.co/models?filter=musicgen
}
class    a   (			lowercase__       ):



      """simple docstring"""


      a  :			Optional[int]								=     'musicgen_decoder'
      a  :			Optional[int]								=     ['past_key_values']
      def __init__(				self :      Optional[Any]		,				__lowercase :      str=2048		,				__lowercase :      Optional[int]=2048		,				__lowercase :      List[Any]=24		,				__lowercase :      Optional[Any]=4096		,				__lowercase :      Dict=16		,				__lowercase :      List[Any]=0.0		,				__lowercase :      Tuple=True		,				__lowercase :      List[Any]="gelu"		,				__lowercase :      Any=1024		,				__lowercase :      Optional[Any]=0.1		,				__lowercase :      List[Any]=0.0		,				__lowercase :      int=0.0		,				__lowercase :      List[str]=0.02		,				__lowercase :      Dict=False		,				__lowercase :      List[str]=4		,				__lowercase :      List[str]=2048		,				__lowercase :      List[Any]=2048		,				__lowercase :      Union[str, Any]=None		,				__lowercase :      List[str]=False		,				**__lowercase :      int		,				)							->				str:
        __UpperCAmelCase							:     Union[str, Any]									=  vocab_size
        __UpperCAmelCase							:     Dict									=  max_position_embeddings
        __UpperCAmelCase							:     str									=  hidden_size
        __UpperCAmelCase							:     Optional[Any]									=  ffn_dim
        __UpperCAmelCase							:     List[Any]									=  num_hidden_layers
        __UpperCAmelCase							:     Any									=  num_attention_heads
        __UpperCAmelCase							:     List[Any]									=  dropout
        __UpperCAmelCase							:     List[str]									=  attention_dropout
        __UpperCAmelCase							:     List[Any]									=  activation_dropout
        __UpperCAmelCase							:     int									=  activation_function
        __UpperCAmelCase							:     Optional[int]									=  initializer_factor
        __UpperCAmelCase							:     List[Any]									=  layerdrop
        __UpperCAmelCase							:     Tuple									=  use_cache
        __UpperCAmelCase							:     Tuple									=  scale_embedding  # scale factor will be sqrt(d_model) if True
        __UpperCAmelCase							:     int									=  num_codebooks
        super().__init__(
            pad_token_id=__lowercase		,				bos_token_id=__lowercase		,				eos_token_id=__lowercase		,				tie_word_embeddings=__lowercase		,				**__lowercase		,				)


class    a   (			lowercase__       ):



      """simple docstring"""


      a  :			Tuple								=     'musicgen'
      a  :			int								=     True
      def __init__(				self :      Any		,				**__lowercase :      Tuple )							->				Union[str, Any]:
        super().__init__(**__lowercase )
        if "text_encoder" not in kwargs or "audio_encoder" not in kwargs or "decoder" not in kwargs:
          raise ValueError("""Config has to be initialized with text_encoder, audio_encoder and decoder config""" )

        __UpperCAmelCase							:     int									=  kwargs.pop("""text_encoder""" )
        __UpperCAmelCase							:     str									=  text_encoder_config.pop("""model_type""" )

        __UpperCAmelCase							:     Tuple									=  kwargs.pop("""audio_encoder""" )
        __UpperCAmelCase							:     Optional[Any]									=  audio_encoder_config.pop("""model_type""" )

        __UpperCAmelCase							:     Optional[Any]									=  kwargs.pop("""decoder""" )

        __UpperCAmelCase							:     Any									=  AutoConfig.for_model(__lowercase		,				**__lowercase )
        __UpperCAmelCase							:     Dict									=  AutoConfig.for_model(__lowercase		,				**__lowercase )
        __UpperCAmelCase							:     Any									=  MusicgenDecoderConfig(**__lowercase )
        __UpperCAmelCase							:     int									=  True
      @classmethod
      def      UpperCAmelCase				(				cls :      Optional[Any]		,				__lowercase :      PretrainedConfig		,				__lowercase :      PretrainedConfig		,				__lowercase :      MusicgenDecoderConfig		,				**__lowercase :      List[Any]		,				)							->				List[str]:
        return cls(
            text_encoder=text_encoder_config.to_dict()		,				audio_encoder=audio_encoder_config.to_dict()		,				decoder=decoder_config.to_dict()		,				**__lowercase		,				)




      def      UpperCAmelCase				(				self :      Any )							->				int:
        __UpperCAmelCase							:     Optional[Any]									=  copy.deepcopy(self.__dict__ )
        __UpperCAmelCase							:     List[Any]									=  self.text_encoder.to_dict()
        __UpperCAmelCase							:     str									=  self.audio_encoder.to_dict()
        __UpperCAmelCase							:     List[str]									=  self.decoder.to_dict()
        __UpperCAmelCase							:     List[str]									=  self.__class__.model_type
        return output


