


from sklearn.metrics import roc_auc_score

import datasets


__snake_case     =		'''\nThis metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model\'s predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance.\n\nThis metric has three separate use cases:\n    - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation.\n    - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label.\n    - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label.\n'''

__snake_case     =		'''\nArgs:\n- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case:\n    - binary: expects an array-like of shape (n_samples,)\n    - multiclass: expects an array-like of shape (n_samples,)\n    - multilabel: expects an array-like of shape (n_samples, n_classes)\n- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case:\n    - binary: expects an array-like of shape (n_samples,)\n    - multiclass: expects an array-like of shape (n_samples, n_classes)\n    - multilabel: expects an array-like of shape (n_samples, n_classes)\n- average (`str`): Type of average, and is ignored in the binary use case. Defaults to \'macro\'. Options are:\n    - `\'micro\'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case.\n    - `\'macro\'`: Calculate metrics for each label, and find their unweighted mean.  This does not take label imbalance into account.\n    - `\'weighted\'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label).\n    - `\'samples\'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case.\n    - `None`:  No average is calculated, and scores for each class are returned. Only works with the multilabels use case.\n- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None.\n- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`.\n- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are:\n    - `\'ovr\'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == \'macro\'`, because class imbalance affects the composition of each of the \'rest\' groupings.\n    - `\'ovo\'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == \'macro\'`.\n- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in\n    `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in\n    `prediction_scores` is used. Defaults to `None`.\nReturns:\n    roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average=\'None\'`. Otherwise, returns `float`.\nExamples:\n    Example 1:\n        >>> roc_auc_score = datasets.load_metric("roc_auc")\n        >>> refs = [1, 0, 1, 1, 0, 0]\n        >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores)\n        >>> print(round(results[\'roc_auc\'], 2))\n        0.78\n\n    Example 2:\n        >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass")\n        >>> refs = [1, 0, 1, 2, 2, 0]\n        >>> pred_scores = [[0.3, 0.5, 0.2],\n        ...                 [0.7, 0.2, 0.1],\n        ...                 [0.005, 0.99, 0.005],\n        ...                 [0.2, 0.3, 0.5],\n        ...                 [0.1, 0.1, 0.8],\n        ...                 [0.1, 0.7, 0.2]]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class=\'ovr\')\n        >>> print(round(results[\'roc_auc\'], 2))\n        0.85\n\n    Example 3:\n        >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel")\n        >>> refs = [[1, 1, 0],\n        ...         [1, 1, 0],\n        ...         [0, 1, 0],\n        ...         [0, 0, 1],\n        ...         [0, 1, 1],\n        ...         [1, 0, 1]]\n        >>> pred_scores = [[0.3, 0.5, 0.2],\n        ...                 [0.7, 0.2, 0.1],\n        ...                 [0.005, 0.99, 0.005],\n        ...                 [0.2, 0.3, 0.5],\n        ...                 [0.1, 0.1, 0.8],\n        ...                 [0.1, 0.7, 0.2]]\n        >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None)\n        >>> print([round(res, 2) for res in results[\'roc_auc\']])\n        [0.83, 0.38, 0.94]\n'''

__snake_case     =		'''\\n@article{doi:10.1177/0272989X8900900307,\nauthor = {Donna Katzman McClish},\ntitle ={Analyzing a Portion of the ROC Curve},\njournal = {Medical Decision Making},\nvolume = {9},\nnumber = {3},\npages = {190-195},\nyear = {1989},\ndoi = {10.1177/0272989X8900900307},\n    note ={PMID: 2668680},\nURL = {https://doi.org/10.1177/0272989X8900900307},\neprint = {https://doi.org/10.1177/0272989X8900900307}\n}\n\n\n@article{10.1023/A:1010920819831,\nauthor = {Hand, David J. and Till, Robert J.},\ntitle = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems},\nyear = {2001},\nissue_date = {November 2001},\npublisher = {Kluwer Academic Publishers},\naddress = {USA},\nvolume = {45},\nnumber = {2},\nissn = {0885-6125},\nurl = {https://doi.org/10.1023/A:1010920819831},\ndoi = {10.1023/A:1010920819831},\njournal = {Mach. Learn.},\nmonth = {oct},\npages = {171–186},\nnumpages = {16},\nkeywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic}\n}\n\n\n@article{scikit-learn,\ntitle={Scikit-learn: Machine Learning in {P}ython},\nauthor={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\njournal={Journal of Machine Learning Research},\nvolume={12},\npages={2825--2830},\nyear={2011}\n}\n'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,     _KWARGS_DESCRIPTION		)
class 					__lowerCamelCase	(datasets.Metric		):







							def 						snake_case_					(	self:				Optional[int]			):






												'''simple docstring'''




												return datasets.MetricInfo(
												    description=_DESCRIPTION,citation=_CITATION,inputs_description=_KWARGS_DESCRIPTION,features=datasets.Features(
												        {
												            'prediction_scores': datasets.Sequence(datasets.Value('float'			)			),
												            'references': datasets.Value('int32'			),
												        }
												        if self.config_name == 'multiclass'
												        else {
												            'references': datasets.Sequence(datasets.Value('int32'			)			),
												            'prediction_scores': datasets.Sequence(datasets.Value('float'			)			),
												        }
												        if self.config_name == 'multilabel'
												        else {
												            'references': datasets.Value('int32'			),
												            'prediction_scores': datasets.Value('float'			),
												        }			),reference_urls=['https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html'],)







							def 						snake_case_					(	self:				List[Any],A_:				Optional[Any],A_:				Optional[Any],A_:				str="macro",A_:				List[str]=None,A_:				List[str]=None,A_:				List[str]="raise",A_:				int=None,):






												'''simple docstring'''




												return {
												    "roc_auc": roc_auc_score(
												        lowercase__,lowercase__,average=lowercase__,sample_weight=lowercase__,max_fpr=lowercase__,multi_class=lowercase__,labels=lowercase__,)
												}






import inspect
import unittest

from transformers import ResNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ResNetBackbone, ResNetForImageClassification, ResNetModel
    from transformers.models.resnet.modeling_resnet import RESNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor



class 					__lowerCamelCase	:







       def __init__(	self:				Dict,A_:				Optional[Any],A_:				str=3,A_:				Any=32,A_:				Optional[Any]=3,A_:				Optional[Any]=10,A_:				str=[10, 20, 30, 40],A_:				Optional[int]=[1, 1, 2, 1],A_:				Dict=True,A_:				Any=True,A_:				Dict="relu",A_:				Union[str, Any]=3,A_:				Optional[int]=None,A_:				Optional[int]=["stage2", "stage3", "stage4"],A_:				Tuple=[2, 3, 4],):






            '''simple docstring'''




            __UpperCamelCase				=	parent
            __UpperCamelCase				=	batch_size
            __UpperCamelCase				=	image_size
            __UpperCamelCase				=	num_channels
            __UpperCamelCase				=	embeddings_size
            __UpperCamelCase				=	hidden_sizes
            __UpperCamelCase				=	depths
            __UpperCamelCase				=	is_training
            __UpperCamelCase				=	use_labels
            __UpperCamelCase				=	hidden_act
            __UpperCamelCase				=	num_labels
            __UpperCamelCase				=	scope
            __UpperCamelCase				=	len(A_			)
            __UpperCamelCase				=	out_features
            __UpperCamelCase				=	out_indices







       def 						snake_case_					(	self:				str			):






            '''simple docstring'''




            __UpperCamelCase				=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

            __UpperCamelCase				=	None
            if self.use_labels:
                 __UpperCamelCase				=	ids_tensor([self.batch_size],self.num_labels			)

            __UpperCamelCase				=	self.get_config()

            return config, pixel_values, labels







       def 						snake_case_					(	self:				Optional[Any]			):






            '''simple docstring'''




            return ResNetConfig(
                num_channels=self.num_channels,embeddings_size=self.embeddings_size,hidden_sizes=self.hidden_sizes,depths=self.depths,hidden_act=self.hidden_act,num_labels=self.num_labels,out_features=self.out_features,out_indices=self.out_indices,)







       def 						snake_case_					(	self:				Optional[int],A_:				int,A_:				Union[str, Any],A_:				List[str]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModel(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)
            # expected last hidden states: B, C, H // 32, W // 32
            self.parent.assertEqual(
                result.last_hidden_state.shape,(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32),)







       def 						snake_case_					(	self:				int,A_:				int,A_:				List[str],A_:				Optional[Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.num_labels
            __UpperCamelCase				=	ResNetForImageClassification(A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_,labels=A_			)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)			)







       def 						snake_case_					(	self:				Optional[Any],A_:				Any,A_:				str,A_:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetBackbone(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)

            # verify feature maps
            self.parent.assertEqual(len(result.feature_maps			),len(config.out_features			)			)
            self.parent.assertListEqual(list(result.feature_maps[0].shape			),[self.batch_size, self.hidden_sizes[1], 4, 4]			)

            # verify channels
            self.parent.assertEqual(len(model.channels			),len(config.out_features			)			)
            self.parent.assertListEqual(model.channels,config.hidden_sizes[1:]			)

            # verify backbone works with out_features=None
            __UpperCamelCase				=	None
            __UpperCamelCase				=	ResNetBackbone(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)

            # verify feature maps
            self.parent.assertEqual(len(result.feature_maps			),1			)
            self.parent.assertListEqual(list(result.feature_maps[0].shape			),[self.batch_size, self.hidden_sizes[-1], 1, 1]			)

            # verify channels
            self.parent.assertEqual(len(model.channels			),1			)
            self.parent.assertListEqual(model.channels,[config.hidden_sizes[-1]]			)







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            __UpperCamelCase				=	self.prepare_config_and_inputs()
            __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase				=	config_and_inputs
            __UpperCamelCase				=	{'pixel_values': pixel_values}
            return config, inputs_dict



@require_torch
class 					__lowerCamelCase	(_a							,     _a							,     unittest.TestCase		):
       _lowercase    							=      (
           (
               ResNetModel,
               ResNetForImageClassification,
               ResNetBackbone,
           )
           if is_torch_available()
           else ()
       )
       _lowercase    							=      (
           {"""feature-extraction""": ResNetModel, """image-classification""": ResNetForImageClassification}
           if is_torch_available()
           else {}
       )

       _lowercase    							=      True
       _lowercase    							=      False
       _lowercase    							=      False
       _lowercase    							=      False
       _lowercase    							=      False







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModelTester(self			)
            __UpperCamelCase				=	ConfigTester(self,config_class=A_,has_text_modality=A_			)







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            self.create_and_test_config_common_properties()
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.create_and_test_config_with_num_labels()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            return







       @unittest.skip(reason='ResNet does not use inputs_embeds'			)
       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            pass







       @unittest.skip(reason='ResNet does not support input and output embeddings'			)
       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            pass







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 __UpperCamelCase				=	model_class(A_			)
                 __UpperCamelCase				=	inspect.signature(model.forward			)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 __UpperCamelCase				=	[*signature.parameters.keys()]

                 __UpperCamelCase				=	['pixel_values']
                 self.assertListEqual(arg_names[:1],A_			)







       def 						snake_case_					(	self:				List[Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*A_			)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_backbone(*A_			)







       def 						snake_case_					(	self:				List[Any]			):






            '''simple docstring'''




            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 __UpperCamelCase				=	model_class(config=A_			)
                 for name, module in model.named_modules():
                      if isinstance(A_,(nn.BatchNormad, nn.GroupNorm)			):
                           self.assertTrue(
                               torch.all(module.weight == 1			),msg=F'''Parameter {name} of model {model_class} seems not properly initialized''',)
                           self.assertTrue(
                               torch.all(module.bias == 0			),msg=F'''Parameter {name} of model {model_class} seems not properly initialized''',)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            def check_hidden_states_output(A_:				Tuple,A_:				Tuple,A_:				Optional[int]			):
                 __UpperCamelCase				=	model_class(A_			)
                 model.to(A_			)
                 model.eval()

                 with torch.no_grad():
                      __UpperCamelCase				=	model(**self._prepare_for_class(A_,A_			)			)

                 __UpperCamelCase				=	outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                 __UpperCamelCase				=	self.model_tester.num_stages
                 self.assertEqual(len(A_			),expected_num_stages + 1			)

                 # ResNet's feature maps are of shape (batch_size, num_channels, height, width)
                 self.assertListEqual(
                     list(hidden_states[0].shape[-2:]			),[self.model_tester.image_size // 4, self.model_tester.image_size // 4],)

            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()
            __UpperCamelCase				=	['basic', 'bottleneck']
            for model_class in self.all_model_classes:
                 for layer_type in layers_type:
                      __UpperCamelCase				=	layer_type
                      __UpperCamelCase				=	True
                      check_hidden_states_output(A_,A_,A_			)

                      # check that output_hidden_states also work using config
                      del inputs_dict["output_hidden_states"]
                      __UpperCamelCase				=	True

                      check_hidden_states_output(A_,A_,A_			)







       @unittest.skip(reason='ResNet does not use feedforward chunking'			)
       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            pass







       def 						snake_case_					(	self:				Any			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*A_			)







       @slow
       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            for model_name in RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __UpperCamelCase				=	ResNetModel.from_pretrained(A_			)
                 self.assertIsNotNone(A_			)






def 						_A						(						)			->				int:


     """simple docstring"""


     __UpperCamelCase				=	Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'				)
     return image



@require_torch
@require_vision
class 					__lowerCamelCase	(unittest.TestCase		):







       @cached_property
       def 						snake_case_					(	self:				Optional[Any]			):






            '''simple docstring'''




            return (
                AutoImageProcessor.from_pretrained(RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]			)
                if is_vision_available()
                else None
            )







       @slow
       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetForImageClassification.from_pretrained(RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]			).to(A_			)

            __UpperCamelCase				=	self.default_image_processor
            __UpperCamelCase				=	prepare_img()
            __UpperCamelCase				=	image_processor(images=A_,return_tensors='pt'			).to(A_			)

            # forward pass
            with torch.no_grad():
                 __UpperCamelCase				=	model(**A_			)

            # verify the logits
            __UpperCamelCase				=	torch.Size((1, 1000)			)
            self.assertEqual(outputs.logits.shape,A_			)

            __UpperCamelCase				=	torch.tensor([-1_1.1_0_6_9, -9.7_8_7_7, -8.3_7_7_7]			).to(A_			)

            self.assertTrue(torch.allclose(outputs.logits[0, :3],A_,atol=1E-4			)			)



@require_torch
class 					__lowerCamelCase	(_a							,     unittest.TestCase		):
       _lowercase    							=      (ResNetBackbone,) if is_torch_available() else ()
       _lowercase    							=      False
       _lowercase    							=      ResNetConfig







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModelTester(self			)


