


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMModel,
        TFXLMWithLMHeadModel,
        XLMConfig,
    )



class 					__lowerCamelCase	:







       def __init__(	self:				Union[str, Any],A_:				List[str],):






            '''simple docstring'''




            __UpperCamelCase				=	parent
            __UpperCamelCase				=	13
            __UpperCamelCase				=	7
            __UpperCamelCase				=	True
            __UpperCamelCase				=	True
            __UpperCamelCase				=	True
            __UpperCamelCase				=	True
            __UpperCamelCase				=	True
            __UpperCamelCase				=	False
            __UpperCamelCase				=	False
            __UpperCamelCase				=	False
            __UpperCamelCase				=	2
            __UpperCamelCase				=	99
            __UpperCamelCase				=	0
            __UpperCamelCase				=	32
            __UpperCamelCase				=	2
            __UpperCamelCase				=	4
            __UpperCamelCase				=	0.1
            __UpperCamelCase				=	0.1
            __UpperCamelCase				=	512
            __UpperCamelCase				=	16
            __UpperCamelCase				=	2
            __UpperCamelCase				=	0.0_2
            __UpperCamelCase				=	3
            __UpperCamelCase				=	4
            __UpperCamelCase				=	'last'
            __UpperCamelCase				=	True
            __UpperCamelCase				=	None
            __UpperCamelCase				=	0







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	ids_tensor([self.batch_size, self.seq_length],self.vocab_size			)
            __UpperCamelCase				=	random_attention_mask([self.batch_size, self.seq_length],dtype=tf.floataa			)

            __UpperCamelCase				=	None
            if self.use_input_lengths:
                 __UpperCamelCase				=	(
                     ids_tensor([self.batch_size],vocab_size=2			) + self.seq_length - 2
                 )  # small variation of seq_length

            __UpperCamelCase				=	None
            if self.use_token_type_ids:
                 __UpperCamelCase				=	ids_tensor([self.batch_size, self.seq_length],self.n_langs			)

            __UpperCamelCase				=	None
            __UpperCamelCase				=	None
            __UpperCamelCase				=	None
            if self.use_labels:
                 __UpperCamelCase				=	ids_tensor([self.batch_size],self.type_sequence_label_size			)
                 __UpperCamelCase				=	ids_tensor([self.batch_size, self.seq_length],self.num_labels			)
                 __UpperCamelCase				=	ids_tensor([self.batch_size],2,dtype=tf.floataa			)
                 __UpperCamelCase				=	ids_tensor([self.batch_size],self.num_choices			)

            __UpperCamelCase				=	XLMConfig(
                vocab_size=self.vocab_size,n_special=self.n_special,emb_dim=self.hidden_size,n_layers=self.num_hidden_layers,n_heads=self.num_attention_heads,dropout=self.hidden_dropout_prob,attention_dropout=self.attention_probs_dropout_prob,gelu_activation=self.gelu_activation,sinusoidal_embeddings=self.sinusoidal_embeddings,asm=self.asm,causal=self.causal,n_langs=self.n_langs,max_position_embeddings=self.max_position_embeddings,initializer_range=self.initializer_range,summary_type=self.summary_type,use_proj=self.use_proj,bos_token_id=self.bos_token_id,)

            return (
                config,
                input_ids,
                token_type_ids,
                input_lengths,
                sequence_labels,
                token_labels,
                is_impossible_labels,
                choice_labels,
                input_mask,
            )







       def 						snake_case_					(	self:				Any,A_:				Optional[int],A_:				int,A_:				Union[str, Any],A_:				Union[str, Any],A_:				List[str],A_:				Tuple,A_:				Optional[int],A_:				Optional[Any],A_:				Any,):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMModel(config=A_			)
            __UpperCamelCase				=	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
            __UpperCamelCase				=	model(A_			)

            __UpperCamelCase				=	[input_ids, input_mask]
            __UpperCamelCase				=	model(A_			)
            self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)			)







       def 						snake_case_					(	self:				Dict,A_:				Dict,A_:				Union[str, Any],A_:				str,A_:				str,A_:				Union[str, Any],A_:				Optional[Any],A_:				Dict,A_:				Optional[int],A_:				List[Any],):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMWithLMHeadModel(A_			)

            __UpperCamelCase				=	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
            __UpperCamelCase				=	model(A_			)

            __UpperCamelCase				=	outputs

            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)			)







       def 						snake_case_					(	self:				Optional[Any],A_:				Union[str, Any],A_:				int,A_:				Dict,A_:				Any,A_:				Dict,A_:				Optional[Any],A_:				Union[str, Any],A_:				str,A_:				int,):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMForQuestionAnsweringSimple(A_			)

            __UpperCamelCase				=	{'input_ids': input_ids, 'lengths': input_lengths}

            __UpperCamelCase				=	model(A_			)

            self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)			)
            self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)			)







       def 						snake_case_					(	self:				Union[str, Any],A_:				Dict,A_:				Tuple,A_:				Any,A_:				Tuple,A_:				int,A_:				Optional[Any],A_:				Tuple,A_:				Optional[int],A_:				List[Any],):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMForSequenceClassification(A_			)

            __UpperCamelCase				=	{'input_ids': input_ids, 'lengths': input_lengths}

            __UpperCamelCase				=	model(A_			)

            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.type_sequence_label_size)			)







       def 						snake_case_					(	self:				List[Any],A_:				List[str],A_:				Dict,A_:				List[Any],A_:				Dict,A_:				Optional[int],A_:				Dict,A_:				Union[str, Any],A_:				int,A_:				List[Any],):






            '''simple docstring'''




            __UpperCamelCase				=	self.num_labels
            __UpperCamelCase				=	TFXLMForTokenClassification(config=A_			)
            __UpperCamelCase				=	{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
            __UpperCamelCase				=	model(A_			)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)			)







       def 						snake_case_					(	self:				List[str],A_:				int,A_:				Union[str, Any],A_:				Tuple,A_:				int,A_:				Any,A_:				Any,A_:				List[str],A_:				Dict,A_:				Optional[int],):






            '''simple docstring'''




            __UpperCamelCase				=	self.num_choices
            __UpperCamelCase				=	TFXLMForMultipleChoice(config=A_			)
            __UpperCamelCase				=	tf.tile(tf.expand_dims(A_,1			),(1, self.num_choices, 1)			)
            __UpperCamelCase				=	tf.tile(tf.expand_dims(A_,1			),(1, self.num_choices, 1)			)
            __UpperCamelCase				=	tf.tile(tf.expand_dims(A_,1			),(1, self.num_choices, 1)			)
            __UpperCamelCase				=	{
                'input_ids': multiple_choice_inputs_ids,
                'attention_mask': multiple_choice_input_mask,
                'token_type_ids': multiple_choice_token_type_ids,
            }
            __UpperCamelCase				=	model(A_			)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)			)







       def 						snake_case_					(	self:				Any			):






            '''simple docstring'''




            __UpperCamelCase				=	self.prepare_config_and_inputs()
            (
                (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  (
                __UpperCamelCase
            ),  
            )				=	config_and_inputs
            __UpperCamelCase				=	{
                'input_ids': input_ids,
                'token_type_ids': token_type_ids,
                'langs': token_type_ids,
                'lengths': input_lengths,
            }
            return config, inputs_dict



@require_tf
class 					__lowerCamelCase	(_a							,     _a							,     unittest.TestCase		):
       _lowercase    							=      (
           (
               TFXLMModel,
               TFXLMWithLMHeadModel,
               TFXLMForSequenceClassification,
               TFXLMForQuestionAnsweringSimple,
               TFXLMForTokenClassification,
               TFXLMForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       _lowercase    							=      (
           (TFXLMWithLMHeadModel,) if is_tf_available() else ()
       )  # TODO (PVP): Check other models whether language generation is also applicable
       _lowercase    							=      (
           {
               """feature-extraction""": TFXLMModel,
               """fill-mask""": TFXLMWithLMHeadModel,
               """question-answering""": TFXLMForQuestionAnsweringSimple,
               """text-classification""": TFXLMForSequenceClassification,
               """text-generation""": TFXLMWithLMHeadModel,
               """token-classification""": TFXLMForTokenClassification,
               """zero-shot""": TFXLMForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       _lowercase    							=      False
       _lowercase    							=      False







       def 						snake_case_					(	self:				Optional[int],A_:				Dict,A_:				Union[str, Any],A_:				str,A_:				int,A_:				str			):






            '''simple docstring'''




            if (
                pipeline_test_casse_name == "QAPipelineTests"
                and tokenizer_name is not None
                and not tokenizer_name.endswith('Fast'			)
            ):
                 # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                 # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                 # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                 return True

            return False







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMModelTester(self			)
            __UpperCamelCase				=	ConfigTester(self,config_class=A_,emb_dim=37			)







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            self.config_tester.run_common_tests()







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_model(*A_			)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_lm_head(*A_			)







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_qa(*A_			)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_sequence_classif(*A_			)







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_for_token_classification(*A_			)







       def 						snake_case_					(	self:				Tuple			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_xlm_for_multiple_choice(*A_			)







       @slow
       def 						snake_case_					(	self:				Any			):






            '''simple docstring'''




            for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __UpperCamelCase				=	TFXLMModel.from_pretrained(A_			)
                 self.assertIsNotNone(A_			)



@require_tf
class 					__lowerCamelCase	(unittest.TestCase		):







       @slow
       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'			)
            __UpperCamelCase				=	tf.convert_to_tensor([[14, 447]],dtype=tf.intaa			)  # the president
            __UpperCamelCase				=	[
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
                14,
                447,
            ]  # the president the president the president the president the president the president the president the president the president the president
            # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
            __UpperCamelCase				=	model.generate(A_,do_sample=A_			)
            self.assertListEqual(output_ids[0].numpy().tolist(),A_			)






import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand



class 					__lowerCamelCase	(_a		):







       @staticmethod
       def 						snake_case_					(	A_:				ArgumentParser			):






            '''simple docstring'''




            __UpperCamelCase				=	parser.add_parser('login',help='Log in using the same credentials as on huggingface.co'			)
            login_parser.set_defaults(func=lambda A_			: LoginCommand(A_			)			)
            __UpperCamelCase				=	parser.add_parser('whoami',help='Find out which huggingface.co account you are logged in as.'			)
            whoami_parser.set_defaults(func=lambda A_			: WhoamiCommand(A_			)			)
            __UpperCamelCase				=	parser.add_parser('logout',help='Log out'			)
            logout_parser.set_defaults(func=lambda A_			: LogoutCommand(A_			)			)

            # new system: git-based repo system
            __UpperCamelCase				=	parser.add_parser(
                'repo',help='Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.',)
            __UpperCamelCase				=	repo_parser.add_subparsers(
                help='Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands'			)
            __UpperCamelCase				=	repo_subparsers.add_parser(
                'create',help='Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co'			)
            repo_create_parser.add_argument(
                'name',type=A_,help='Name for your model\'s repo. Will be namespaced under your username to build the model id.',)
            repo_create_parser.add_argument('--organization',type=A_,help='Optional: organization namespace.'			)
            repo_create_parser.add_argument('-y','--yes',action='store_true',help='Optional: answer Yes to the prompt'			)
            repo_create_parser.set_defaults(func=lambda A_			: RepoCreateCommand(A_			)			)



class 					__lowerCamelCase	:
       _lowercase    							=      """\u001b[1m"""
       _lowercase    							=      """\u001b[31m"""
       _lowercase    							=      """\u001b[90m"""
       _lowercase    							=      """\u001b[0m"""







       @classmethod
       def 						snake_case_					(	cls:				Optional[Any],A_:				Tuple			):






            '''simple docstring'''




            return F'''{cls._bold}{s}{cls._reset}'''







       @classmethod
       def 						snake_case_					(	cls:				Any,A_:				Union[str, Any]			):






            '''simple docstring'''




            return F'''{cls._bold}{cls._red}{s}{cls._reset}'''







       @classmethod
       def 						snake_case_					(	cls:				Optional[Any],A_:				Dict			):






            '''simple docstring'''




            return F'''{cls._gray}{s}{cls._reset}'''






def 						_A						(						_lowercase			,       _lowercase				)			->				str:


     """simple docstring"""


     __UpperCamelCase				=	[max(len(str(_lowercase				)				) for x in col				) for col in zip(*_lowercase			,       _lowercase				)]
     __UpperCamelCase				=	('{{:{}}} ' * len(_lowercase				)).format(*_lowercase				)
     __UpperCamelCase				=	[]
     lines.append(row_format.format(*_lowercase				)				)
     lines.append(row_format.format(*['-' * w for w in col_widths]				)				)
     for row in rows:
          lines.append(row_format.format(*_lowercase				)				)
     return "\n".join(_lowercase				)



class 					__lowerCamelCase	:







       def __init__(	self:				List[Any],A_:				Any			):






            '''simple docstring'''




            __UpperCamelCase				=	args



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'ERROR! `huggingface-cli login` uses an outdated login mechanism '
                    'that is not compatible with the Hugging Face Hub backend anymore. '
                    'Please use `huggingface-cli login instead.'			)			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '
                    '`huggingface-cli whoami` instead.'			)			)
            __UpperCamelCase				=	HfFolder.get_token()
            if token is None:
                 print('Not logged in'			)
                 exit()
            try:
                 __UpperCamelCase,  __UpperCamelCase				=	whoami(A_			)
                 print(A_			)
                 if orgs:
                      print(ANSI.bold('orgs: '			),','.join(A_			)			)
            except HTTPError as e:
                 print(A_			)
                 print(ANSI.red(e.response.text			)			)
                 exit(1			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'ERROR! `transformers-cli logout` uses an outdated logout mechanism '
                    'that is not compatible with the Hugging Face Hub backend anymore. '
                    'Please use `huggingface-cli logout instead.'			)			)



class 					__lowerCamelCase	(_a		):







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            print(
                ANSI.red(
                    'WARNING! Managing repositories through transformers-cli is deprecated. '
                    'Please use `huggingface-cli` instead.'			)			)
            __UpperCamelCase				=	HfFolder.get_token()
            if token is None:
                 print('Not logged in'			)
                 exit(1			)
            try:
                 __UpperCamelCase				=	subprocess.check_output(['git', '--version']			).decode('utf-8'			)
                 print(ANSI.gray(stdout.strip()			)			)
            except FileNotFoundError:
                 print('Looks like you do not have git installed, please install.'			)

            try:
                 __UpperCamelCase				=	subprocess.check_output(['git-lfs', '--version']			).decode('utf-8'			)
                 print(ANSI.gray(stdout.strip()			)			)
            except FileNotFoundError:
                 print(
                     ANSI.red(
                         'Looks like you do not have git-lfs installed, please install.'
                         ' You can install from https://git-lfs.github.com/.'
                         ' Then run `git lfs install` (you only have to do this once).'			)			)
            print(''			)

            __UpperCamelCase,  __UpperCamelCase				=	whoami(A_			)
            __UpperCamelCase				=	self.args.organization if self.args.organization is not None else user
            __UpperCamelCase				=	F'''{namespace}/{self.args.name}'''
            print(F'''You are about to create {ANSI.bold(A_			)}'''			)

            if not self.args.yes:
                 __UpperCamelCase				=	input('Proceed? [Y/n] '			).lower()
                 if not (choice == "" or choice == "y" or choice == "yes"):
                      print('Abort'			)
                      exit()
            try:
                 __UpperCamelCase				=	create_repo(A_,name=self.args.name,organization=self.args.organization			)
            except HTTPError as e:
                 print(A_			)
                 print(ANSI.red(e.response.text			)			)
                 exit(1			)
            print('\nYour repo now lives at:'			)
            print(F'''  {ANSI.bold(A_			)}'''			)
            print('\nYou can clone it locally with the command below, and commit/push as usual.'			)
            print(F'''\n  git clone {url}'''			)
            print(''			)


