






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class        lowercase__       (     _UpperCAmelCase,   unittest.TestCase		):
	a_             =ElectraTokenizer
	a_             =ElectraTokenizerFast
	a_             =True
	a_             =True
	a_             =filter_non_english

	def 						UpperCAmelCase					(							self    )->   Any:







							'''simple docstring'''





							super().setUp()

							lowerCAmelCase__	 =  [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							lowerCAmelCase__	 =  os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES["vocab_file"]    )
							with open(self.vocab_file						,      "w"						,      encoding="utf-8"    ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )

	def 						UpperCAmelCase					(							self						,      __UpperCAmelCase    )->   Dict:







							'''simple docstring'''





							lowerCAmelCase__	 =  "UNwant\u00E9d,running"
							lowerCAmelCase__	 =  "unwanted, running"
							return input_text, output_text

	def 						UpperCAmelCase					(							self    )->   Tuple:







							'''simple docstring'''





							lowerCAmelCase__	 =  self.tokenizer_class(self.vocab_file    )

							lowerCAmelCase__	 =  tokenizer.tokenize("UNwant\u00E9d,running"    )
							self.assertListEqual(__UpperCAmelCase						,      ["un", "##want", "##ed", ",", "runn", "##ing"]    )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase    )						,      [9, 6, 7, 12, 10, 11]    )

	def 						UpperCAmelCase					(							self    )->   Optional[Any]:







							'''simple docstring'''





							if not self.test_rust_tokenizer:
													return

							lowerCAmelCase__	 =  self.get_tokenizer()
							lowerCAmelCase__	 =  self.get_rust_tokenizer()

							lowerCAmelCase__	 =  "UNwant\u00E9d,running"

							lowerCAmelCase__	 =  tokenizer.tokenize(__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.tokenize(__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

							lowerCAmelCase__	 =  tokenizer.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

							lowerCAmelCase__	 =  self.get_rust_tokenizer()
							lowerCAmelCase__	 =  tokenizer.encode(__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.encode(__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

							# With lower casing
							lowerCAmelCase__	 =  self.get_tokenizer(do_lower_case=__UpperCAmelCase    )
							lowerCAmelCase__	 =  self.get_rust_tokenizer(do_lower_case=__UpperCAmelCase    )

							lowerCAmelCase__	 =  "UNwant\u00E9d,running"

							lowerCAmelCase__	 =  tokenizer.tokenize(__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.tokenize(__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

							lowerCAmelCase__	 =  tokenizer.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

							lowerCAmelCase__	 =  self.get_rust_tokenizer()
							lowerCAmelCase__	 =  tokenizer.encode(__UpperCAmelCase    )
							lowerCAmelCase__	 =  rust_tokenizer.encode(__UpperCAmelCase    )
							self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

	def 						UpperCAmelCase					(							self    )->   Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"    )						,      ["ah", "\u535A", "\u63A8", "zz"]    )

	def 						UpperCAmelCase					(							self    )->   Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )						,      ["hello", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )						,      ["hello"]    )

	def 						UpperCAmelCase					(							self    )->   List[str]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase						,      strip_accents=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )						,      ["hällo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )						,      ["h\u00E9llo"]    )

	def 						UpperCAmelCase					(							self    )->   Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase						,      strip_accents=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )						,      ["hallo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )						,      ["hello"]    )

	def 						UpperCAmelCase					(							self    )->   int:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )						,      ["hallo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )						,      ["hello"]    )

	def 						UpperCAmelCase					(							self    )->   Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )						,      ["HeLLo", "!", "how", "Are", "yoU", "?"]    )

	def 						UpperCAmelCase					(							self    )->   str:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase						,      strip_accents=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )						,      ["HäLLo", "!", "how", "Are", "yoU", "?"]    )

	def 						UpperCAmelCase					(							self    )->   List[Any]:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase						,      strip_accents=__UpperCAmelCase    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )						,      ["HaLLo", "!", "how", "Are", "yoU", "?"]    )

	def 						UpperCAmelCase					(							self    )->   int:







							'''simple docstring'''





							lowerCAmelCase__	 =  BasicTokenizer(do_lower_case=__UpperCAmelCase						,      never_split=["[UNK]"]    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"    )						,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]    )

	def 						UpperCAmelCase					(							self    )->   Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__	 =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							lowerCAmelCase__	 =  {}
							for i, token in enumerate(__UpperCAmelCase    ):
													lowerCAmelCase__	 =  i
							lowerCAmelCase__	 =  WordpieceTokenizer(vocab=__UpperCAmelCase						,      unk_token="[UNK]"    )

							self.assertListEqual(tokenizer.tokenize(""    )						,      []    )

							self.assertListEqual(tokenizer.tokenize("unwanted running"    )						,      ["un", "##want", "##ed", "runn", "##ing"]    )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"    )						,      ["[UNK]", "runn", "##ing"]    )

	def 						UpperCAmelCase					(							self    )->   List[str]:







							'''simple docstring'''





							self.assertTrue(_is_whitespace(" "    )    )
							self.assertTrue(_is_whitespace("\t"    )    )
							self.assertTrue(_is_whitespace("\r"    )    )
							self.assertTrue(_is_whitespace("\n"    )    )
							self.assertTrue(_is_whitespace("\u00A0"    )    )

							self.assertFalse(_is_whitespace("A"    )    )
							self.assertFalse(_is_whitespace("-"    )    )

	def 						UpperCAmelCase					(							self    )->   List[Any]:







							'''simple docstring'''





							self.assertTrue(_is_control("\u0005"    )    )

							self.assertFalse(_is_control("A"    )    )
							self.assertFalse(_is_control(" "    )    )
							self.assertFalse(_is_control("\t"    )    )
							self.assertFalse(_is_control("\r"    )    )

	def 						UpperCAmelCase					(							self    )->   Union[str, Any]:







							'''simple docstring'''





							self.assertTrue(_is_punctuation("-"    )    )
							self.assertTrue(_is_punctuation("$"    )    )
							self.assertTrue(_is_punctuation("`"    )    )
							self.assertTrue(_is_punctuation("."    )    )

							self.assertFalse(_is_punctuation("A"    )    )
							self.assertFalse(_is_punctuation(" "    )    )

	def 						UpperCAmelCase					(							self    )->   List[str]:







							'''simple docstring'''





							lowerCAmelCase__	 =  self.get_tokenizer()
							lowerCAmelCase__	 =  self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__UpperCAmelCase    ) for t in ["Test", "\xad", "test"]]						,      [["[UNK]"], [], ["[UNK]"]]    )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__UpperCAmelCase    ) for t in ["Test", "\xad", "test"]]						,      [["[UNK]"], [], ["[UNK]"]]    )

	@slow
	def 						UpperCAmelCase					(							self    )->   str:







							'''simple docstring'''





							lowerCAmelCase__	 =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"    )

							lowerCAmelCase__	 =  tokenizer.encode("sequence builders"						,      add_special_tokens=__UpperCAmelCase    )
							lowerCAmelCase__	 =  tokenizer.encode("multi-sequence build"						,      add_special_tokens=__UpperCAmelCase    )

							lowerCAmelCase__	 =  tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase    )
							lowerCAmelCase__	 =  tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase						,      __UpperCAmelCase    )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]

	def 						UpperCAmelCase					(							self    )->   List[Any]:







							'''simple docstring'''





							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
																			lowerCAmelCase__	 =  self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase						,      **__UpperCAmelCase    )

																			lowerCAmelCase__	 =  F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																			lowerCAmelCase__	 =  tokenizer_r.encode_plus(
																			    __UpperCAmelCase						,      return_attention_mask=__UpperCAmelCase						,      return_token_type_ids=__UpperCAmelCase						,      return_offsets_mapping=__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase						,      )

																			lowerCAmelCase__	 =  tokenizer_r.do_lower_case if hasattr(__UpperCAmelCase						,      "do_lower_case"    ) else False
																			lowerCAmelCase__	 =  (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "A"),
																			        ((1, 2), ","),
																			        ((3, 5), "na"),
																			        ((5, 6), "##ï"),
																			        ((6, 8), "##ve"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "Allen"),
																			        ((21, 23), "##NL"),
																			        ((23, 24), "##P"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "a"),
																			        ((1, 2), ","),
																			        ((3, 8), "naive"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "allen"),
																			        ((21, 23), "##nl"),
																			        ((23, 24), "##p"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]						,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]    )    )
																			self.assertEqual([e[0] for e in expected_results]						,      tokens["offset_mapping"]    )


	def 						UpperCAmelCase					(							self    )->   str:







							'''simple docstring'''





							lowerCAmelCase__	 =  ["的", "人", "有"]
							lowerCAmelCase__	 =  "".join(__UpperCAmelCase    )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
																			lowerCAmelCase__	 =  True
																			lowerCAmelCase__	 =  self.tokenizer_class.from_pretrained(__UpperCAmelCase						,      **__UpperCAmelCase    )
																			lowerCAmelCase__	 =  self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase						,      **__UpperCAmelCase    )

																			lowerCAmelCase__	 =  tokenizer_p.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
																			lowerCAmelCase__	 =  tokenizer_r.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )

																			lowerCAmelCase__	 =  tokenizer_r.convert_ids_to_tokens(__UpperCAmelCase    )
																			lowerCAmelCase__	 =  tokenizer_p.convert_ids_to_tokens(__UpperCAmelCase    )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )
																			self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )

																			lowerCAmelCase__	 =  False
																			lowerCAmelCase__	 =  self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase						,      **__UpperCAmelCase    )
																			lowerCAmelCase__	 =  self.tokenizer_class.from_pretrained(__UpperCAmelCase						,      **__UpperCAmelCase    )

																			lowerCAmelCase__	 =  tokenizer_r.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )
																			lowerCAmelCase__	 =  tokenizer_p.encode(__UpperCAmelCase						,      add_special_tokens=__UpperCAmelCase    )

																			lowerCAmelCase__	 =  tokenizer_r.convert_ids_to_tokens(__UpperCAmelCase    )
																			lowerCAmelCase__	 =  tokenizer_p.convert_ids_to_tokens(__UpperCAmelCase    )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			lowerCAmelCase__	 =  [
																			    F"##{token}" if idx != 0 else token for idx, token in enumerate(__UpperCAmelCase    )
																			]
																			self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )
																			self.assertListEqual(__UpperCAmelCase						,      __UpperCAmelCase    )














import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


a_							=      logging.get_logger(__name__)

a_							=      {
    '''facebook/musicgen-small''': '''https://huggingface.co/facebook/musicgen-small/resolve/main/config.json''',
    # See all Musicgen models at https://huggingface.co/models?filter=musicgen
}


class        lowercase__       (     _UpperCAmelCase		):
	a_             ="""musicgen_decoder"""
	a_             =["""past_key_values"""]

	def __init__(							self						,      __UpperCAmelCase=2048						,      __UpperCAmelCase=2048						,      __UpperCAmelCase=24						,      __UpperCAmelCase=4096						,      __UpperCAmelCase=16						,      __UpperCAmelCase=0.0						,      __UpperCAmelCase=True						,      __UpperCAmelCase="gelu"						,      __UpperCAmelCase=1024						,      __UpperCAmelCase=0.1						,      __UpperCAmelCase=0.0						,      __UpperCAmelCase=0.0						,      __UpperCAmelCase=0.02						,      __UpperCAmelCase=False						,      __UpperCAmelCase=4						,      __UpperCAmelCase=2048						,      __UpperCAmelCase=2048						,      __UpperCAmelCase=None						,      __UpperCAmelCase=False						,      **__UpperCAmelCase						,      )->   Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__	 =  vocab_size
							lowerCAmelCase__	 =  max_position_embeddings
							lowerCAmelCase__	 =  hidden_size
							lowerCAmelCase__	 =  ffn_dim
							lowerCAmelCase__	 =  num_hidden_layers
							lowerCAmelCase__	 =  num_attention_heads
							lowerCAmelCase__	 =  dropout
							lowerCAmelCase__	 =  attention_dropout
							lowerCAmelCase__	 =  activation_dropout
							lowerCAmelCase__	 =  activation_function
							lowerCAmelCase__	 =  initializer_factor
							lowerCAmelCase__	 =  layerdrop
							lowerCAmelCase__	 =  use_cache
							lowerCAmelCase__	 =  scale_embedding  # scale factor will be sqrt(d_model) if True
							lowerCAmelCase__	 =  num_codebooks
							super().__init__(
							    pad_token_id=__UpperCAmelCase						,      bos_token_id=__UpperCAmelCase						,      eos_token_id=__UpperCAmelCase						,      tie_word_embeddings=__UpperCAmelCase						,      **__UpperCAmelCase						,      )





class        lowercase__       (     _UpperCAmelCase		):
	a_             ="""musicgen"""
	a_             =True

	def __init__(							self						,      **__UpperCAmelCase    )->   Any:







							'''simple docstring'''





							super().__init__(**__UpperCAmelCase    )
							if "text_encoder" not in kwargs or "audio_encoder" not in kwargs or "decoder" not in kwargs:
													raise ValueError("Config has to be initialized with text_encoder, audio_encoder and decoder config"    )

							lowerCAmelCase__	 =  kwargs.pop("text_encoder"    )
							lowerCAmelCase__	 =  text_encoder_config.pop("model_type"    )

							lowerCAmelCase__	 =  kwargs.pop("audio_encoder"    )
							lowerCAmelCase__	 =  audio_encoder_config.pop("model_type"    )

							lowerCAmelCase__	 =  kwargs.pop("decoder"    )

							lowerCAmelCase__	 =  AutoConfig.for_model(__UpperCAmelCase						,      **__UpperCAmelCase    )
							lowerCAmelCase__	 =  AutoConfig.for_model(__UpperCAmelCase						,      **__UpperCAmelCase    )
							lowerCAmelCase__	 =  MusicgenDecoderConfig(**__UpperCAmelCase    )
							lowerCAmelCase__	 =  True

	@classmethod
	def 						UpperCAmelCase					(							cls						,      __UpperCAmelCase						,      __UpperCAmelCase						,      __UpperCAmelCase						,      **__UpperCAmelCase						,      )->   Tuple:







							'''simple docstring'''





							return cls(
							    text_encoder=text_encoder_config.to_dict()						,      audio_encoder=audio_encoder_config.to_dict()						,      decoder=decoder_config.to_dict()						,      **__UpperCAmelCase						,      )


	def 						UpperCAmelCase					(							self    )->   Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__	 =  copy.deepcopy(self.__dict__    )
							lowerCAmelCase__	 =  self.text_encoder.to_dict()
							lowerCAmelCase__	 =  self.audio_encoder.to_dict()
							lowerCAmelCase__	 =  self.decoder.to_dict()
							lowerCAmelCase__	 =  self.__class__.model_type
							return output






