

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 		a_	(							A__			,      unittest.TestCase					):



     """simple docstring"""







     __UpperCAmelCase      		=				PreTrainedTokenizerFast
     __UpperCAmelCase      		=				False
     __UpperCAmelCase      		=				True
     __UpperCAmelCase      		=				'tokenizer_file'
     def 	_lowerCAmelCase					(				self					:							List[Any]    ):
          SCREAMING_SNAKE_CASE										=False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          SCREAMING_SNAKE_CASE										=True

          SCREAMING_SNAKE_CASE										=['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
          SCREAMING_SNAKE_CASE										='''SaulLu/dummy-tokenizer-bytelevel-bpe'''

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          SCREAMING_SNAKE_CASE										=[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          SCREAMING_SNAKE_CASE										=PreTrainedTokenizerFast.from_pretrained(model_paths[0]    )
          tokenizer.save_pretrained(self.tmpdirname    )
     def 	_lowerCAmelCase					(				self					:							Tuple    ):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass
     def 	_lowerCAmelCase					(				self					:							Dict    ):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass
     def 	_lowerCAmelCase					(				self					:							List[Any]    ):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass
     def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass
     def 	_lowerCAmelCase					(				self					:							Union[str, Any]    ):
          SCREAMING_SNAKE_CASE										=self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
                    try:
                         SCREAMING_SNAKE_CASE										=tempfile.mkdtemp()
                         SCREAMING_SNAKE_CASE										=self.rust_tokenizer_class.from_pretrained(__snake_case      ,**__snake_case    )

                         tokenizer.save_pretrained(self.tmpdirname    )
                         super().test_training_new_tokenizer()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname    )
                         SCREAMING_SNAKE_CASE										=tmpdirname_orig
     def 	_lowerCAmelCase					(				self					:							Any    ):
          SCREAMING_SNAKE_CASE										=self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
                    try:
                         SCREAMING_SNAKE_CASE										=tempfile.mkdtemp()
                         SCREAMING_SNAKE_CASE										=self.rust_tokenizer_class.from_pretrained(__snake_case      ,**__snake_case    )

                         tokenizer.save_pretrained(self.tmpdirname    )
                         super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname    )
                         SCREAMING_SNAKE_CASE										=tmpdirname_orig


     def 	_lowerCAmelCase					(				self					:							List[Any]    ):
          SCREAMING_SNAKE_CASE										=self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name    )

          SCREAMING_SNAKE_CASE										=('''a''' for _ in range(1000    ))
          SCREAMING_SNAKE_CASE										=tokenizer.train_new_from_iterator(text_iterator=__snake_case      ,length=1000      ,vocab_size=50    )

          SCREAMING_SNAKE_CASE										=new_tokenizer.encode('a🤗'    )
          self.assertEqual(__snake_case      ,[64, 172, 253, 97, 245]    )







@require_tokenizers
class 		a_	(							unittest.TestCase					):



     """simple docstring"""







     def 	_lowerCAmelCase					(				self					:							Tuple    ):
          SCREAMING_SNAKE_CASE										=AutoTokenizer.from_pretrained('bert-base-cased'    )
          SCREAMING_SNAKE_CASE										=json.loads(tokenizer._tokenizer.to_str()    )
          SCREAMING_SNAKE_CASE										=len(__snake_case    )

          with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               SCREAMING_SNAKE_CASE										=['''tokenizer.4.0.0.json''']
               tokenizer.save_pretrained(__snake_case    )
               json.dump(__snake_case      ,open(os.path.join(__snake_case      ,'tokenizer.4.0.0.json'    )      ,'w'    )    )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               SCREAMING_SNAKE_CASE										=AutoTokenizer.from_pretrained(__snake_case    )
               self.assertEqual(len(__snake_case    )      ,len(__snake_case    ) + 1    )
               SCREAMING_SNAKE_CASE										=json.loads(new_tokenizer._tokenizer.to_str()    )
               self.assertIn('huggingface'      ,json_tokenizer['model']['vocab']    )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(__snake_case      ,'tokenizer.4.0.0.json'    )      ,os.path.join(__snake_case      ,'tokenizer.42.0.0.json'    )    )
               SCREAMING_SNAKE_CASE										=['''tokenizer.42.0.0.json''']
               tokenizer.save_pretrained(__snake_case    )
               SCREAMING_SNAKE_CASE										=AutoTokenizer.from_pretrained(__snake_case    )
               self.assertEqual(len(__snake_case    )      ,len(__snake_case    )    )
               SCREAMING_SNAKE_CASE										=json.loads(new_tokenizer._tokenizer.to_str()    )
               self.assertNotIn('huggingface'      ,json_tokenizer['model']['vocab']    )


     def 	_lowerCAmelCase					(				self					:							Dict    ):
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          SCREAMING_SNAKE_CASE										='''hf-internal-testing/test-two-tokenizers'''

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          SCREAMING_SNAKE_CASE										=AutoTokenizer.from_pretrained(__snake_case    )
          self.assertEqual(len(__snake_case    )      ,28997    )
          SCREAMING_SNAKE_CASE										=json.loads(tokenizer._tokenizer.to_str()    )
          self.assertIn('huggingface'      ,json_tokenizer['model']['vocab']    )

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          SCREAMING_SNAKE_CASE										='''3.0.0'''
          SCREAMING_SNAKE_CASE										=old_transformers.models.auto.AutoTokenizer.from_pretrained(__snake_case    )
          self.assertEqual(len(__snake_case    )      ,28996    )
          SCREAMING_SNAKE_CASE										=json.loads(old_tokenizer._tokenizer.to_str()    )
          self.assertNotIn('huggingface'      ,json_tokenizer['model']['vocab']    )







@require_tokenizers
class 		a_	(							unittest.TestCase					):



     """simple docstring"""







     def 	_lowerCAmelCase					(				self					:							List[str]    ):
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          SCREAMING_SNAKE_CASE										=PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'    )
          SCREAMING_SNAKE_CASE										='''The Matrix is a 1999 science fiction action film.'''

          with concurrent.futures.ThreadPoolExecutor() as executor:
               SCREAMING_SNAKE_CASE										=[executor.submit(self.fetch      ,__snake_case      ,__snake_case    ) for i in range(10    )]
               SCREAMING_SNAKE_CASE										=[future.result() for future in futures]
               self.assertEqual(__snake_case      ,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10    )]    )


     def 	_lowerCAmelCase					(				self					:							Tuple      ,snake_case					:							List[str]      ,snake_case					:							Dict    ):
          return tokenizer.encode(__snake_case      ,truncation='longest_first'      ,padding='longest'    )




from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers.models.mpnet.modeling_tf_mpnet import (
     TFMPNetForMaskedLM,
     TFMPNetForMultipleChoice,
     TFMPNetForQuestionAnswering,
     TFMPNetForSequenceClassification,
     TFMPNetForTokenClassification,
     TFMPNetModel,
 )



class 		a_	:



      """simple docstring"""







      def __init__(				self					:							Any      ,snake_case					:							List[Any]      ,snake_case					:							Optional[Any]=13      ,snake_case					:							str=7      ,snake_case					:							List[Any]=True      ,snake_case					:							Optional[Any]=True      ,snake_case					:							int=False      ,snake_case					:							List[str]=True      ,snake_case					:							Optional[int]=99      ,snake_case					:							Any=64      ,snake_case					:							int=2      ,snake_case					:							str=4      ,snake_case					:							str=64      ,snake_case					:							List[Any]="gelu"      ,snake_case					:							Dict=0.1      ,snake_case					:							int=0.1      ,snake_case					:							List[Any]=512      ,snake_case					:							int=16      ,snake_case					:							List[Any]=2      ,snake_case					:							Union[str, Any]=0.02      ,snake_case					:							List[str]=3      ,snake_case					:							Any=4      ,snake_case					:							Tuple=None      ,):
           SCREAMING_SNAKE_CASE										=parent
           SCREAMING_SNAKE_CASE										=batch_size
           SCREAMING_SNAKE_CASE										=seq_length
           SCREAMING_SNAKE_CASE										=is_training
           SCREAMING_SNAKE_CASE										=use_input_mask
           SCREAMING_SNAKE_CASE										=use_token_type_ids
           SCREAMING_SNAKE_CASE										=use_labels
           SCREAMING_SNAKE_CASE										=vocab_size
           SCREAMING_SNAKE_CASE										=hidden_size
           SCREAMING_SNAKE_CASE										=num_hidden_layers
           SCREAMING_SNAKE_CASE										=num_attention_heads
           SCREAMING_SNAKE_CASE										=intermediate_size
           SCREAMING_SNAKE_CASE										=hidden_act
           SCREAMING_SNAKE_CASE										=hidden_dropout_prob
           SCREAMING_SNAKE_CASE										=attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE										=max_position_embeddings
           SCREAMING_SNAKE_CASE										=type_vocab_size
           SCREAMING_SNAKE_CASE										=type_sequence_label_size
           SCREAMING_SNAKE_CASE										=initializer_range
           SCREAMING_SNAKE_CASE										=num_labels
           SCREAMING_SNAKE_CASE										=num_choices
           SCREAMING_SNAKE_CASE										=scope
      def 	_lowerCAmelCase					(				self					:							List[Any]    ):
           SCREAMING_SNAKE_CASE										=ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size    )

           SCREAMING_SNAKE_CASE										=None
           if self.use_input_mask:
                SCREAMING_SNAKE_CASE										=random_attention_mask([self.batch_size, self.seq_length]    )

           SCREAMING_SNAKE_CASE										=None
           SCREAMING_SNAKE_CASE										=None
           SCREAMING_SNAKE_CASE										=None
           if self.use_labels:
                SCREAMING_SNAKE_CASE										=ids_tensor([self.batch_size]      ,self.type_sequence_label_size    )
                SCREAMING_SNAKE_CASE										=ids_tensor([self.batch_size, self.seq_length]      ,self.num_labels    )
                SCREAMING_SNAKE_CASE										=ids_tensor([self.batch_size]      ,self.num_choices    )

           SCREAMING_SNAKE_CASE										=MPNetConfig(
               vocab_size=self.vocab_size      ,hidden_size=self.hidden_size      ,num_hidden_layers=self.num_hidden_layers      ,num_attention_heads=self.num_attention_heads      ,intermediate_size=self.intermediate_size      ,hidden_act=self.hidden_act      ,hidden_dropout_prob=self.hidden_dropout_prob      ,attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,max_position_embeddings=self.max_position_embeddings      ,initializer_range=self.initializer_range      ,)
           return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
      def 	_lowerCAmelCase					(				self					:							Dict      ,snake_case					:							Any      ,snake_case					:							Optional[int]      ,snake_case					:							str      ,snake_case					:							Union[str, Any]      ,snake_case					:							Optional[Any]      ,snake_case					:							Any    ):
           SCREAMING_SNAKE_CASE										=TFMPNetModel(config=snake_case    )
           SCREAMING_SNAKE_CASE										={'input_ids': input_ids, 'attention_mask': input_mask}
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           SCREAMING_SNAKE_CASE										=[input_ids, input_mask]
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.hidden_size)    )
      def 	_lowerCAmelCase					(				self					:							List[Any]      ,snake_case					:							List[str]      ,snake_case					:							Tuple      ,snake_case					:							Dict      ,snake_case					:							Optional[Any]      ,snake_case					:							Optional[int]      ,snake_case					:							Dict    ):
           SCREAMING_SNAKE_CASE										=TFMPNetForMaskedLM(config=snake_case    )
           SCREAMING_SNAKE_CASE										={'input_ids': input_ids, 'attention_mask': input_mask}
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.vocab_size)    )
      def 	_lowerCAmelCase					(				self					:							int      ,snake_case					:							List[str]      ,snake_case					:							Dict      ,snake_case					:							List[Any]      ,snake_case					:							Optional[int]      ,snake_case					:							Optional[int]      ,snake_case					:							Tuple    ):
           SCREAMING_SNAKE_CASE										=TFMPNetForQuestionAnswering(config=snake_case    )
           SCREAMING_SNAKE_CASE										={
               'input_ids': input_ids,
               'attention_mask': input_mask,
           }
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.start_logits.shape      ,(self.batch_size, self.seq_length)    )
           self.parent.assertEqual(result.end_logits.shape      ,(self.batch_size, self.seq_length)    )
      def 	_lowerCAmelCase					(				self					:							List[Any]      ,snake_case					:							Union[str, Any]      ,snake_case					:							str      ,snake_case					:							str      ,snake_case					:							Any      ,snake_case					:							Tuple      ,snake_case					:							Optional[Any]    ):
           SCREAMING_SNAKE_CASE										=self.num_labels
           SCREAMING_SNAKE_CASE										=TFMPNetForSequenceClassification(snake_case    )
           SCREAMING_SNAKE_CASE										={'input_ids': input_ids, 'attention_mask': input_mask}
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_labels)    )
      def 	_lowerCAmelCase					(				self					:							int      ,snake_case					:							Any      ,snake_case					:							Union[str, Any]      ,snake_case					:							int      ,snake_case					:							List[str]      ,snake_case					:							Tuple      ,snake_case					:							Union[str, Any]    ):
           SCREAMING_SNAKE_CASE										=self.num_choices
           SCREAMING_SNAKE_CASE										=TFMPNetForMultipleChoice(snake_case    )
           SCREAMING_SNAKE_CASE										=tf.tile(tf.expand_dims(snake_case      ,1    )      ,(1, self.num_choices, 1)    )
           SCREAMING_SNAKE_CASE										=tf.tile(tf.expand_dims(snake_case      ,1    )      ,(1, self.num_choices, 1)    )
           SCREAMING_SNAKE_CASE										={
               'input_ids': multiple_choice_inputs_ids,
               'attention_mask': multiple_choice_input_mask,
           }
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_choices)    )
      def 	_lowerCAmelCase					(				self					:							Any      ,snake_case					:							Tuple      ,snake_case					:							Dict      ,snake_case					:							Tuple      ,snake_case					:							Union[str, Any]      ,snake_case					:							Optional[int]      ,snake_case					:							Dict    ):
           SCREAMING_SNAKE_CASE										=self.num_labels
           SCREAMING_SNAKE_CASE										=TFMPNetForTokenClassification(snake_case    )
           SCREAMING_SNAKE_CASE										={'input_ids': input_ids, 'attention_mask': input_mask}
           SCREAMING_SNAKE_CASE										=model(snake_case    )
           self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.num_labels)    )


      def 	_lowerCAmelCase					(				self					:							Optional[int]    ):
           SCREAMING_SNAKE_CASE										=self.prepare_config_and_inputs()
           ((SCREAMING_SNAKE_CASE)						,  (SCREAMING_SNAKE_CASE)						,  (SCREAMING_SNAKE_CASE)						,  (SCREAMING_SNAKE_CASE)						,  (SCREAMING_SNAKE_CASE)						,  (SCREAMING_SNAKE_CASE))										=config_and_inputs
           SCREAMING_SNAKE_CASE										={'input_ids': input_ids, 'attention_mask': input_mask}
           return config, inputs_dict







@require_tf
class 		a_	(							lowerCamelCase_			,      lowerCamelCase_			,      unittest.TestCase					):



      """simple docstring"""







      __UpperCAmelCase      		=				(
          (
              TFMPNetForMaskedLM,
              TFMPNetForMultipleChoice,
              TFMPNetForQuestionAnswering,
              TFMPNetForSequenceClassification,
              TFMPNetForTokenClassification,
              TFMPNetModel,
          )
          if is_tf_available()
          else ()
      )
      __UpperCAmelCase      		=				(
          {
              'feature-extraction': TFMPNetModel,
              'fill-mask': TFMPNetForMaskedLM,
              'question-answering': TFMPNetForQuestionAnswering,
              'text-classification': TFMPNetForSequenceClassification,
              'token-classification': TFMPNetForTokenClassification,
              'zero-shot': TFMPNetForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      __UpperCAmelCase      		=				False
      __UpperCAmelCase      		=				False
      def 	_lowerCAmelCase					(				self					:							Dict    ):
           SCREAMING_SNAKE_CASE										=TFMPNetModelTester(self    )
           SCREAMING_SNAKE_CASE										=ConfigTester(self      ,config_class=snake_case      ,hidden_size=37    )
      def 	_lowerCAmelCase					(				self					:							Dict    ):
           self.config_tester.run_common_tests()
      def 	_lowerCAmelCase					(				self					:							Union[str, Any]    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_model(*snake_case    )
      def 	_lowerCAmelCase					(				self					:							List[Any]    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_masked_lm(*snake_case    )
      def 	_lowerCAmelCase					(				self					:							Dict    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_question_answering(*snake_case    )
      def 	_lowerCAmelCase					(				self					:							str    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_sequence_classification(*snake_case    )
      def 	_lowerCAmelCase					(				self					:							Dict    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_multiple_choice(*snake_case    )
      def 	_lowerCAmelCase					(				self					:							List[str]    ):
           SCREAMING_SNAKE_CASE										=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_mpnet_for_token_classification(*snake_case    )


      @slow
      def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
           for model_name in ["microsoft/mpnet-base"]:
                SCREAMING_SNAKE_CASE										=TFMPNetModel.from_pretrained(snake_case    )
                self.assertIsNotNone(snake_case    )







@require_tf
class 		a_	(							unittest.TestCase					):



      """simple docstring"""







      @slow
      def 	_lowerCAmelCase					(				self					:							Optional[Any]    ):
           SCREAMING_SNAKE_CASE										=TFMPNetModel.from_pretrained('microsoft/mpnet-base'    )
           SCREAMING_SNAKE_CASE										=tf.constant([[0, 1, 2, 3, 4, 5]]    )
           SCREAMING_SNAKE_CASE										=model(snake_case    )[0]

           SCREAMING_SNAKE_CASE										=[1, 6, 768]
           self.assertEqual(output.shape      ,snake_case    )

           SCREAMING_SNAKE_CASE										=tf.constant(
               [
                   [
                       [-0.1_067_172, 0.08_216_473, 0.0_024_543],
                       [-0.03_465_879, 0.8_354_118, -0.03_252_288],
                       [-0.06_569_476, -0.12_424_111, -0.0_494_436],
                   ]
               ]    )
           tf.debugging.assert_near(output[:, :3, :3]      ,snake_case      ,atol=1e-4    )

