


import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode






def 						_A						(						_lowercase			,       **_lowercase				)			->				Dict:


     """simple docstring"""


     return RunBeamCommand(
         args.dataset			,       args.name			,       args.cache_dir			,       args.beam_pipeline_options			,       args.data_dir			,       args.all_configs			,       args.save_info or args.save_infos			,       args.ignore_verifications			,       args.force_redownload			,       **_lowercase			,       )



class 					__lowerCamelCase	(_a		):







       @staticmethod
       def 						snake_case_					(	A_:				ArgumentParser			):






            '''simple docstring'''




            __UpperCamelCase				=	parser.add_parser('run_beam',help='Run a Beam dataset processing pipeline'			)
            run_beam_parser.add_argument('dataset',type=A_,help='Name of the dataset to download'			)
            run_beam_parser.add_argument('--name',type=A_,default=A_,help='Dataset config name'			)
            run_beam_parser.add_argument(
                '--cache_dir',type=A_,default=A_,help='Cache directory where the datasets are stored',)
            run_beam_parser.add_argument(
                '--beam_pipeline_options',type=A_,default='',help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`',)
            run_beam_parser.add_argument(
                '--data_dir',type=A_,default=A_,help='Can be used to specify a manual directory to get the files from',)
            run_beam_parser.add_argument('--all_configs',action='store_true',help='Test all dataset configurations'			)
            run_beam_parser.add_argument('--save_info',action='store_true',help='Save the dataset infos file'			)
            run_beam_parser.add_argument(
                '--ignore_verifications',action='store_true',help='Run the test without checksums and splits checks'			)
            run_beam_parser.add_argument('--force_redownload',action='store_true',help='Force dataset redownload'			)
            # aliases
            run_beam_parser.add_argument('--save_infos',action='store_true',help='alias for save_info'			)
            run_beam_parser.set_defaults(func=A_			)







       def __init__(	self:				Optional[int],A_:				str,A_:				str,A_:				str,A_:				str,A_:				str,A_:				bool,A_:				bool,A_:				bool,A_:				bool,**A_:				int,):






            '''simple docstring'''




            __UpperCamelCase				=	dataset
            __UpperCamelCase				=	name
            __UpperCamelCase				=	cache_dir
            __UpperCamelCase				=	beam_pipeline_options
            __UpperCamelCase				=	data_dir
            __UpperCamelCase				=	all_configs
            __UpperCamelCase				=	save_infos
            __UpperCamelCase				=	ignore_verifications
            __UpperCamelCase				=	force_redownload
            __UpperCamelCase				=	config_kwargs







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            import apache_beam as beam

            if self._name is not None and self._all_configs:
                 print('Both parameters `name` and `all_configs` can\'t be used at once.'			)
                 exit(1			)
            __UpperCamelCase,  __UpperCamelCase				=	self._dataset, self._name
            __UpperCamelCase				=	dataset_module_factory(A_			)
            __UpperCamelCase				=	import_main_class(dataset_module.module_path			)
            __UpperCamelCase				=	[]
            if self._beam_pipeline_options:
                 __UpperCamelCase				=	beam.options.pipeline_options.PipelineOptions(
                     flags=[F'''--{opt.strip()}''' for opt in self._beam_pipeline_options.split(','			) if opt]			)
            else:
                 __UpperCamelCase				=	None
            if self._all_configs and len(builder_cls.BUILDER_CONFIGS			) > 0:
                 for builder_config in builder_cls.BUILDER_CONFIGS:
                      builders.append(
                          builder_cls(
                              config_name=builder_config.name,data_dir=self._data_dir,hash=dataset_module.hash,beam_options=A_,cache_dir=self._cache_dir,base_path=dataset_module.builder_kwargs.get('base_path'			),)			)
            else:
                 builders.append(
                     builder_cls(
                         config_name=A_,data_dir=self._data_dir,beam_options=A_,cache_dir=self._cache_dir,base_path=dataset_module.builder_kwargs.get('base_path'			),**self._config_kwargs,)			)

            for builder in builders:
                 builder.download_and_prepare(
                     download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                     if not self._force_redownload
                     else DownloadMode.FORCE_REDOWNLOAD,download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH			),verification_mode=VerificationMode.NO_CHECKS
                     if self._ignore_verifications
                     else VerificationMode.ALL_CHECKS,try_from_hf_gcs=A_,)
                 if self._save_infos:
                      builder._save_infos()

            print('Apache beam run successful.'			)

            # If save_infos=True, the dataset infos file is created next to the loaded module file.
            # Let's move it to the original directory of the dataset script, to allow the user to
            # upload them on S3 at the same time afterwards.
            if self._save_infos:
                 __UpperCamelCase				=	os.path.join(builder_cls.get_imported_module_dir(),config.DATASETDICT_INFOS_FILENAME			)

                 __UpperCamelCase				=	Path(A_			).name + '.py'

                 __UpperCamelCase				=	os.path.join(A_,A_			)
                 if os.path.isfile(A_			):
                      __UpperCamelCase				=	os.path.dirname(A_			)
                 elif os.path.isfile(A_			):
                      __UpperCamelCase				=	path
                 else:  # in case of a remote dataset
                      print(F'''Dataset Infos file saved at {dataset_infos_path}'''			)
                      exit(1			)

                 # Move datasetinfo back to the user
                 __UpperCamelCase				=	os.path.join(A_,config.DATASETDICT_INFOS_FILENAME			)
                 copyfile(A_,A_			)
                 print(F'''Dataset Infos file saved at {user_dataset_infos_path}'''			)






import inspect
import unittest

from transformers import ResNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ResNetBackbone, ResNetForImageClassification, ResNetModel
    from transformers.models.resnet.modeling_resnet import RESNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor



class 					__lowerCamelCase	:







       def __init__(	self:				Dict,A_:				Optional[Any],A_:				str=3,A_:				Any=32,A_:				Optional[Any]=3,A_:				Optional[Any]=10,A_:				str=[10, 20, 30, 40],A_:				Optional[int]=[1, 1, 2, 1],A_:				Dict=True,A_:				Any=True,A_:				Dict="relu",A_:				Union[str, Any]=3,A_:				Optional[int]=None,A_:				Optional[int]=["stage2", "stage3", "stage4"],A_:				Tuple=[2, 3, 4],):






            '''simple docstring'''




            __UpperCamelCase				=	parent
            __UpperCamelCase				=	batch_size
            __UpperCamelCase				=	image_size
            __UpperCamelCase				=	num_channels
            __UpperCamelCase				=	embeddings_size
            __UpperCamelCase				=	hidden_sizes
            __UpperCamelCase				=	depths
            __UpperCamelCase				=	is_training
            __UpperCamelCase				=	use_labels
            __UpperCamelCase				=	hidden_act
            __UpperCamelCase				=	num_labels
            __UpperCamelCase				=	scope
            __UpperCamelCase				=	len(A_			)
            __UpperCamelCase				=	out_features
            __UpperCamelCase				=	out_indices







       def 						snake_case_					(	self:				str			):






            '''simple docstring'''




            __UpperCamelCase				=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

            __UpperCamelCase				=	None
            if self.use_labels:
                 __UpperCamelCase				=	ids_tensor([self.batch_size],self.num_labels			)

            __UpperCamelCase				=	self.get_config()

            return config, pixel_values, labels







       def 						snake_case_					(	self:				Optional[Any]			):






            '''simple docstring'''




            return ResNetConfig(
                num_channels=self.num_channels,embeddings_size=self.embeddings_size,hidden_sizes=self.hidden_sizes,depths=self.depths,hidden_act=self.hidden_act,num_labels=self.num_labels,out_features=self.out_features,out_indices=self.out_indices,)







       def 						snake_case_					(	self:				Optional[int],A_:				int,A_:				Union[str, Any],A_:				List[str]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModel(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)
            # expected last hidden states: B, C, H // 32, W // 32
            self.parent.assertEqual(
                result.last_hidden_state.shape,(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32),)







       def 						snake_case_					(	self:				int,A_:				int,A_:				List[str],A_:				Optional[Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.num_labels
            __UpperCamelCase				=	ResNetForImageClassification(A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_,labels=A_			)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)			)







       def 						snake_case_					(	self:				Optional[Any],A_:				Any,A_:				str,A_:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetBackbone(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)

            # verify feature maps
            self.parent.assertEqual(len(result.feature_maps			),len(config.out_features			)			)
            self.parent.assertListEqual(list(result.feature_maps[0].shape			),[self.batch_size, self.hidden_sizes[1], 4, 4]			)

            # verify channels
            self.parent.assertEqual(len(model.channels			),len(config.out_features			)			)
            self.parent.assertListEqual(model.channels,config.hidden_sizes[1:]			)

            # verify backbone works with out_features=None
            __UpperCamelCase				=	None
            __UpperCamelCase				=	ResNetBackbone(config=A_			)
            model.to(A_			)
            model.eval()
            __UpperCamelCase				=	model(A_			)

            # verify feature maps
            self.parent.assertEqual(len(result.feature_maps			),1			)
            self.parent.assertListEqual(list(result.feature_maps[0].shape			),[self.batch_size, self.hidden_sizes[-1], 1, 1]			)

            # verify channels
            self.parent.assertEqual(len(model.channels			),1			)
            self.parent.assertListEqual(model.channels,[config.hidden_sizes[-1]]			)







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            __UpperCamelCase				=	self.prepare_config_and_inputs()
            __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase				=	config_and_inputs
            __UpperCamelCase				=	{'pixel_values': pixel_values}
            return config, inputs_dict



@require_torch
class 					__lowerCamelCase	(_a							,     _a							,     unittest.TestCase		):
       _lowercase    							=      (
           (
               ResNetModel,
               ResNetForImageClassification,
               ResNetBackbone,
           )
           if is_torch_available()
           else ()
       )
       _lowercase    							=      (
           {"""feature-extraction""": ResNetModel, """image-classification""": ResNetForImageClassification}
           if is_torch_available()
           else {}
       )

       _lowercase    							=      True
       _lowercase    							=      False
       _lowercase    							=      False
       _lowercase    							=      False
       _lowercase    							=      False







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModelTester(self			)
            __UpperCamelCase				=	ConfigTester(self,config_class=A_,has_text_modality=A_			)







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            self.create_and_test_config_common_properties()
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.create_and_test_config_with_num_labels()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()







       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            return







       @unittest.skip(reason='ResNet does not use inputs_embeds'			)
       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            pass







       @unittest.skip(reason='ResNet does not support input and output embeddings'			)
       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            pass







       def 						snake_case_					(	self:				int			):






            '''simple docstring'''




            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 __UpperCamelCase				=	model_class(A_			)
                 __UpperCamelCase				=	inspect.signature(model.forward			)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 __UpperCamelCase				=	[*signature.parameters.keys()]

                 __UpperCamelCase				=	['pixel_values']
                 self.assertListEqual(arg_names[:1],A_			)







       def 						snake_case_					(	self:				List[Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*A_			)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_backbone(*A_			)







       def 						snake_case_					(	self:				List[Any]			):






            '''simple docstring'''




            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 __UpperCamelCase				=	model_class(config=A_			)
                 for name, module in model.named_modules():
                      if isinstance(A_,(nn.BatchNormad, nn.GroupNorm)			):
                           self.assertTrue(
                               torch.all(module.weight == 1			),msg=F'''Parameter {name} of model {model_class} seems not properly initialized''',)
                           self.assertTrue(
                               torch.all(module.bias == 0			),msg=F'''Parameter {name} of model {model_class} seems not properly initialized''',)







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            def check_hidden_states_output(A_:				Tuple,A_:				Tuple,A_:				Optional[int]			):
                 __UpperCamelCase				=	model_class(A_			)
                 model.to(A_			)
                 model.eval()

                 with torch.no_grad():
                      __UpperCamelCase				=	model(**self._prepare_for_class(A_,A_			)			)

                 __UpperCamelCase				=	outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                 __UpperCamelCase				=	self.model_tester.num_stages
                 self.assertEqual(len(A_			),expected_num_stages + 1			)

                 # ResNet's feature maps are of shape (batch_size, num_channels, height, width)
                 self.assertListEqual(
                     list(hidden_states[0].shape[-2:]			),[self.model_tester.image_size // 4, self.model_tester.image_size // 4],)

            __UpperCamelCase,  __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs_for_common()
            __UpperCamelCase				=	['basic', 'bottleneck']
            for model_class in self.all_model_classes:
                 for layer_type in layers_type:
                      __UpperCamelCase				=	layer_type
                      __UpperCamelCase				=	True
                      check_hidden_states_output(A_,A_,A_			)

                      # check that output_hidden_states also work using config
                      del inputs_dict["output_hidden_states"]
                      __UpperCamelCase				=	True

                      check_hidden_states_output(A_,A_,A_			)







       @unittest.skip(reason='ResNet does not use feedforward chunking'			)
       def 						snake_case_					(	self:				List[str]			):






            '''simple docstring'''




            pass







       def 						snake_case_					(	self:				Any			):






            '''simple docstring'''




            __UpperCamelCase				=	self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*A_			)







       @slow
       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            for model_name in RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __UpperCamelCase				=	ResNetModel.from_pretrained(A_			)
                 self.assertIsNotNone(A_			)






def 						_A						(						)			->				int:


     """simple docstring"""


     __UpperCamelCase				=	Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'				)
     return image



@require_torch
@require_vision
class 					__lowerCamelCase	(unittest.TestCase		):







       @cached_property
       def 						snake_case_					(	self:				Optional[Any]			):






            '''simple docstring'''




            return (
                AutoImageProcessor.from_pretrained(RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]			)
                if is_vision_available()
                else None
            )







       @slow
       def 						snake_case_					(	self:				Optional[int]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetForImageClassification.from_pretrained(RESNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]			).to(A_			)

            __UpperCamelCase				=	self.default_image_processor
            __UpperCamelCase				=	prepare_img()
            __UpperCamelCase				=	image_processor(images=A_,return_tensors='pt'			).to(A_			)

            # forward pass
            with torch.no_grad():
                 __UpperCamelCase				=	model(**A_			)

            # verify the logits
            __UpperCamelCase				=	torch.Size((1, 1000)			)
            self.assertEqual(outputs.logits.shape,A_			)

            __UpperCamelCase				=	torch.tensor([-1_1.1_0_6_9, -9.7_8_7_7, -8.3_7_7_7]			).to(A_			)

            self.assertTrue(torch.allclose(outputs.logits[0, :3],A_,atol=1E-4			)			)



@require_torch
class 					__lowerCamelCase	(_a							,     unittest.TestCase		):
       _lowercase    							=      (ResNetBackbone,) if is_torch_available() else ()
       _lowercase    							=      False
       _lowercase    							=      ResNetConfig







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	ResNetModelTester(self			)


