"""simple docstring"""





import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							import PIL

							from transformers import FlavaImageProcessor
							from transformers.image_utils import PILImageResampling
							from transformers.models.flava.image_processing_flava import (
							    FLAVA_CODEBOOK_MEAN,
							    FLAVA_CODEBOOK_STD,
							    FLAVA_IMAGE_MEAN,
							    FLAVA_IMAGE_STD,
							)
else:
							_UpperCamelCase       :						List[Any]     =   None


class 					UpperCAmelCase_     (   unittest.TestCase):


						def __init__(       self		,      a		,      a=7		,      a=3		,      a=3_0		,      a=4_0_0		,      a=True		,      a=None		,      a=True		,      a=None		,      a=None		,      a=True		,      a=1 / 2_5_5		,      a=True		,      a=FLAVA_IMAGE_MEAN		,      a=FLAVA_IMAGE_STD		,      a=1_4		,      a=7_5		,      a=None		,      a=1_6		,      a=0.3		,      a=None		,      a=True		,      a=None		,      a=None		,      a=True		,      a=None		,      a=True		,      a=True		,      a=FLAVA_CODEBOOK_MEAN		,      a=FLAVA_CODEBOOK_STD		,      )	-> List[Any]:
													lowercase__      :		Union[str, Any]          =     size if size is not None else {'height': 2_2_4, 'width': 2_2_4}
													lowercase__      :		List[str]          =     crop_size if crop_size is not None else {'height': 2_2_4, 'width': 2_2_4}
													lowercase__      :		Dict          =     codebook_size if codebook_size is not None else {'height': 1_1_2, 'width': 1_1_2}
													lowercase__      :		Optional[Any]          =     codebook_crop_size if codebook_crop_size is not None else {'height': 1_1_2, 'width': 1_1_2}

													lowercase__      :		Union[str, Any]          =     parent
													lowercase__      :		str          =     batch_size
													lowercase__      :		List[Any]          =     num_channels
													lowercase__      :		Any          =     do_resize
													lowercase__      :		Tuple          =     do_rescale
													lowercase__      :		Optional[Any]          =     rescale_factor
													lowercase__      :		Any          =     min_resolution
													lowercase__      :		int          =     max_resolution
													lowercase__      :		int          =     size
													lowercase__      :		Any          =     resample if resample is not None else PILImageResampling.BICUBIC
													lowercase__      :		Dict          =     do_normalize
													lowercase__      :		Optional[Any]          =     image_mean
													lowercase__      :		int          =     image_std
													lowercase__      :		int          =     do_center_crop
													lowercase__      :		Union[str, Any]          =     crop_size

													lowercase__      :		Union[str, Any]          =     input_size_patches
													lowercase__      :		List[str]          =     total_mask_patches
													lowercase__      :		Tuple          =     mask_group_max_patches
													lowercase__      :		List[str]          =     mask_group_min_patches
													lowercase__      :		str          =     mask_group_min_aspect_ratio
													lowercase__      :		Optional[Any]          =     mask_group_max_aspect_ratio

													lowercase__      :		Dict          =     codebook_do_resize
													lowercase__      :		Any          =     codebook_size
													lowercase__      :		Optional[Any]          =     codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
													lowercase__      :		Optional[int]          =     codebook_do_center_crop
													lowercase__      :		Union[str, Any]          =     codebook_crop_size
													lowercase__      :		int          =     codebook_do_map_pixels
													lowercase__      :		Union[str, Any]          =     codebook_do_normalize
													lowercase__      :		Tuple          =     codebook_image_mean
													lowercase__      :		str          =     codebook_image_std


						def 		_UpperCAmelCase	(       self							)	-> Dict:
													return {
													    "image_mean": self.image_mean,
													    "image_std": self.image_std,
													    "do_normalize": self.do_normalize,
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "resample": self.resample,
													    "do_rescale": self.do_rescale,
													    "rescale_factor": self.rescale_factor,
													    "do_center_crop": self.do_center_crop,
													    "crop_size": self.crop_size,
													    "input_size_patches": self.input_size_patches,
													    "total_mask_patches": self.total_mask_patches,
													    "mask_group_max_patches": self.mask_group_max_patches,
													    "mask_group_min_patches": self.mask_group_min_patches,
													    "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
													    "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
													    "codebook_do_resize": self.codebook_do_resize,
													    "codebook_size": self.codebook_size,
													    "codebook_resample": self.codebook_resample,
													    "codebook_do_center_crop": self.codebook_do_center_crop,
													    "codebook_crop_size": self.codebook_crop_size,
													    "codebook_do_map_pixels": self.codebook_do_map_pixels,
													    "codebook_do_normalize": self.codebook_do_normalize,
													    "codebook_image_mean": self.codebook_image_mean,
													    "codebook_image_std": self.codebook_image_std,
													}


						def 		_UpperCAmelCase	(       self							)	-> str:
													return (self.size["height"], self.size["width"])


						def 		_UpperCAmelCase	(       self							)	-> List[Any]:
													return (
													    (self.input_size_patches, self.input_size_patches)
													    if not isinstance(self.input_size_patches		,      a							)
													    else self.input_size_patches
													)


						def 		_UpperCAmelCase	(       self							)	-> Tuple:
													return (self.codebook_size["height"], self.codebook_size["width"])





@require_torch
@require_vision
class 					UpperCAmelCase_     (   A__			,							unittest.TestCase):
						lowerCamelCase__		:   List[str]			  =  FlavaImageProcessor if is_vision_available() else None
						lowerCamelCase__		:   Union[str, Any]			  =  None


						def 		_UpperCAmelCase	(       self							)	-> Union[str, Any]:
													lowercase__      :		Dict          =     FlavaImageProcessingTester(self							)


						@property
						def 		_UpperCAmelCase	(       self							)	-> Tuple:
													return self.image_processor_tester.prepare_image_processor_dict()


						def 		_UpperCAmelCase	(       self							)	-> List[Any]:
													lowercase__      :		Optional[Any]          =     self.image_processing_class(**self.image_processor_dict							)
													self.assertTrue(hasattr(a		,      'image_mean'							)							)
													self.assertTrue(hasattr(a		,      'image_std'							)							)
													self.assertTrue(hasattr(a		,      'do_normalize'							)							)
													self.assertTrue(hasattr(a		,      'do_resize'							)							)
													self.assertTrue(hasattr(a		,      'resample'							)							)
													self.assertTrue(hasattr(a		,      'crop_size'							)							)
													self.assertTrue(hasattr(a		,      'do_center_crop'							)							)
													self.assertTrue(hasattr(a		,      'do_rescale'							)							)
													self.assertTrue(hasattr(a		,      'rescale_factor'							)							)
													self.assertTrue(hasattr(a		,      'masking_generator'							)							)
													self.assertTrue(hasattr(a		,      'codebook_do_resize'							)							)
													self.assertTrue(hasattr(a		,      'codebook_size'							)							)
													self.assertTrue(hasattr(a		,      'codebook_resample'							)							)
													self.assertTrue(hasattr(a		,      'codebook_do_center_crop'							)							)
													self.assertTrue(hasattr(a		,      'codebook_crop_size'							)							)
													self.assertTrue(hasattr(a		,      'codebook_do_map_pixels'							)							)
													self.assertTrue(hasattr(a		,      'codebook_do_normalize'							)							)
													self.assertTrue(hasattr(a		,      'codebook_image_mean'							)							)
													self.assertTrue(hasattr(a		,      'codebook_image_std'							)							)


						def 		_UpperCAmelCase	(       self							)	-> List[Any]:
													lowercase__      :		Tuple          =     self.image_processing_class.from_dict(self.image_processor_dict							)
													self.assertEqual(image_processor.size		,      {'height': 2_2_4, 'width': 2_2_4}							)
													self.assertEqual(image_processor.crop_size		,      {'height': 2_2_4, 'width': 2_2_4}							)
													self.assertEqual(image_processor.codebook_size		,      {'height': 1_1_2, 'width': 1_1_2}							)
													self.assertEqual(image_processor.codebook_crop_size		,      {'height': 1_1_2, 'width': 1_1_2}							)

													lowercase__      :		str          =     self.image_processing_class.from_dict(
													    self.image_processor_dict		,      size=4_2		,      crop_size=8_4		,      codebook_size=3_3		,      codebook_crop_size=6_6							)
													self.assertEqual(image_processor.size		,      {'height': 4_2, 'width': 4_2}							)
													self.assertEqual(image_processor.crop_size		,      {'height': 8_4, 'width': 8_4}							)
													self.assertEqual(image_processor.codebook_size		,      {'height': 3_3, 'width': 3_3}							)
													self.assertEqual(image_processor.codebook_crop_size		,      {'height': 6_6, 'width': 6_6}							)


						def 		_UpperCAmelCase	(       self							)	-> Dict:
													pass


						def 		_UpperCAmelCase	(       self							)	-> int:
													lowercase__      :		str          =     self.image_processing_class(**self.image_processor_dict							)
													# create random PIL images
													lowercase__      :		Any          =     prepare_image_inputs(self.image_processor_tester		,      equal_resolution=a							)
													for image in image_inputs:
																				self.assertIsInstance(a		,      PIL.Image.Image							)

													# Test not batched input
													lowercase__      :		Tuple          =     image_processing(image_inputs[0]		,      return_tensors='pt'							)

													# Test no bool masked pos
													self.assertFalse('bool_masked_pos' in encoded_images							)

													lowercase__   ,				lowercase__      :		str          =     self.image_processor_tester.get_expected_image_size()

													self.assertEqual(
													    encoded_images.pixel_values.shape		,      (1, self.image_processor_tester.num_channels, expected_height, expected_width)		,      )

													# Test batched
													lowercase__      :		List[str]          =     image_processing(a		,      return_tensors='pt'							)
													lowercase__   ,				lowercase__      :		List[Any]          =     self.image_processor_tester.get_expected_image_size()

													# Test no bool masked pos
													self.assertFalse('bool_masked_pos' in encoded_images							)

													self.assertEqual(
													    encoded_images.pixel_values.shape		,      (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,      )


						def 		_UpperCAmelCase	(       self		,      a		,      a							)	-> Any:
													lowercase__      :		List[str]          =     self.image_processing_class(**self.image_processor_dict							)
													# create random tensors
													lowercase__      :		List[str]          =     prepare_image_inputs(self.image_processor_tester		,      equal_resolution=a		,      **a							)
													for image in image_inputs:
																				self.assertIsInstance(a		,      a							)

													# Test not batched input
													lowercase__      :		Dict          =     image_processing(image_inputs[0]		,      return_tensors='pt'							)

													lowercase__   ,				lowercase__      :		Dict          =     self.image_processor_tester.get_expected_image_size()
													self.assertEqual(
													    encoded_images.pixel_values.shape		,      (1, self.image_processor_tester.num_channels, expected_height, expected_width)		,      )

													lowercase__      :		Tuple          =     image_processing(a		,      return_image_mask=a		,      return_tensors='pt'							)

													lowercase__   ,				lowercase__      :		Dict          =     self.image_processor_tester.get_expected_image_size()
													self.assertEqual(
													    encoded_images.pixel_values.shape		,      (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,      )

													lowercase__   ,				lowercase__      :		Any          =     self.image_processor_tester.get_expected_mask_size()
													self.assertEqual(
													    encoded_images.bool_masked_pos.shape		,      (
													        self.image_processor_tester.batch_size,
													        expected_height,
													        expected_width,
													    )		,      )

													# Test batched
													lowercase__      :		str          =     image_processing(a		,      return_tensors='pt'							).pixel_values

													lowercase__   ,				lowercase__      :		Dict          =     self.image_processor_tester.get_expected_image_size()
													self.assertEqual(
													    encoded_images.shape		,      (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,      )

													# Test masking
													lowercase__      :		Tuple          =     image_processing(a		,      return_image_mask=a		,      return_tensors='pt'							)

													lowercase__   ,				lowercase__      :		Optional[Any]          =     self.image_processor_tester.get_expected_image_size()
													self.assertEqual(
													    encoded_images.pixel_values.shape		,      (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,      )

													lowercase__   ,				lowercase__      :		Dict          =     self.image_processor_tester.get_expected_mask_size()
													self.assertEqual(
													    encoded_images.bool_masked_pos.shape		,      (
													        self.image_processor_tester.batch_size,
													        expected_height,
													        expected_width,
													    )		,      )


						def 		_UpperCAmelCase	(       self							)	-> str:
													self._test_call_framework(np.ndarray		,      prepare_kwargs={'numpify': True}							)


						def 		_UpperCAmelCase	(       self							)	-> Optional[int]:
													self._test_call_framework(torch.Tensor		,      prepare_kwargs={'torchify': True}							)


						def 		_UpperCAmelCase	(       self							)	-> List[str]:
													random.seed(1_2_3_4							)
													lowercase__      :		Any          =     self.image_processing_class(**self.image_processor_dict							)
													lowercase__      :		str          =     prepare_image_inputs(self.image_processor_tester		,      equal_resolution=a		,      torchify=a							)

													# Test not batched input
													lowercase__      :		str          =     image_processing(image_inputs[0]		,      return_image_mask=a		,      return_tensors='pt'							)
													self.assertEqual(encoded_images.bool_masked_pos.sum().item()		,      7_5							)


						def 		_UpperCAmelCase	(       self							)	-> int:
													lowercase__      :		Any          =     self.image_processing_class(**self.image_processor_dict							)
													# create random PIL images
													lowercase__      :		Any          =     prepare_image_inputs(self.image_processor_tester		,      equal_resolution=a							)
													for image in image_inputs:
																				self.assertIsInstance(a		,      PIL.Image.Image							)

													# Test not batched input
													lowercase__      :		Union[str, Any]          =     image_processing(image_inputs[0]		,      return_codebook_pixels=a		,      return_tensors='pt'							)
													lowercase__   ,				lowercase__      :		Optional[Any]          =     self.image_processor_tester.get_expected_codebook_image_size()
													self.assertEqual(
													    encoded_images.codebook_pixel_values.shape		,      (1, self.image_processor_tester.num_channels, expected_height, expected_width)		,      )

													# Test batched
													lowercase__      :		Tuple          =     image_processing(a		,      return_codebook_pixels=a		,      return_tensors='pt'							)
													lowercase__   ,				lowercase__      :		Tuple          =     self.image_processor_tester.get_expected_codebook_image_size()
													self.assertEqual(
													    encoded_images.codebook_pixel_values.shape		,      (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,      )


"""simple docstring"""





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode





def     a_					(				_lowerCAmelCase  :			Tuple       ,			**_lowerCAmelCase  :			Any					):





       '''simple docstring'''

       return RunBeamCommand(
           args.dataset       ,			args.name       ,			args.cache_dir       ,			args.beam_pipeline_options       ,			args.data_dir       ,			args.all_configs       ,			args.save_info or args.save_infos       ,			args.ignore_verifications       ,			args.force_redownload       ,			**_lowerCAmelCase       ,			)






class 					UpperCAmelCase_     (   _a):


      @staticmethod
      def 		_UpperCAmelCase	(       a							)	-> Optional[int]:
             lowercase__      :		Optional[int]          =     parser.add_parser('run_beam'		,      help='Run a Beam dataset processing pipeline'							)
             run_beam_parser.add_argument('dataset'		,      type=a		,      help='Name of the dataset to download'							)
             run_beam_parser.add_argument('--name'		,      type=a		,      default=a		,      help='Dataset config name'							)
             run_beam_parser.add_argument(
                 '--cache_dir'		,      type=a		,      default=a		,      help='Cache directory where the datasets are stored'		,      )
             run_beam_parser.add_argument(
                 '--beam_pipeline_options'		,      type=a		,      default=''		,      help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'		,      )
             run_beam_parser.add_argument(
                 '--data_dir'		,      type=a		,      default=a		,      help='Can be used to specify a manual directory to get the files from'		,      )
             run_beam_parser.add_argument('--all_configs'		,      action='store_true'		,      help='Test all dataset configurations'							)
             run_beam_parser.add_argument('--save_info'		,      action='store_true'		,      help='Save the dataset infos file'							)
             run_beam_parser.add_argument(
                 '--ignore_verifications'		,      action='store_true'		,      help='Run the test without checksums and splits checks'							)
             run_beam_parser.add_argument('--force_redownload'		,      action='store_true'		,      help='Force dataset redownload'							)
             # aliases
             run_beam_parser.add_argument('--save_infos'		,      action='store_true'		,      help='alias for save_info'							)
             run_beam_parser.set_defaults(func=a							)


      def __init__(       self		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      **a		,      )	-> Any:
             lowercase__      :		List[Any]          =     dataset
             lowercase__      :		List[Any]          =     name
             lowercase__      :		List[Any]          =     cache_dir
             lowercase__      :		Any          =     beam_pipeline_options
             lowercase__      :		List[str]          =     data_dir
             lowercase__      :		int          =     all_configs
             lowercase__      :		Optional[Any]          =     save_infos
             lowercase__      :		List[Any]          =     ignore_verifications
             lowercase__      :		int          =     force_redownload
             lowercase__      :		List[Any]          =     config_kwargs


      def 		_UpperCAmelCase	(       self							)	-> int:
             import apache_beam as beam

             if self._name is not None and self._all_configs:
                    print('Both parameters `name` and `all_configs` can\'t be used at once.'							)
                    exit(1							)
             lowercase__   ,				lowercase__      :		Optional[int]          =     self._dataset, self._name
             lowercase__      :		List[str]          =     dataset_module_factory(a							)
             lowercase__      :		Dict          =     import_main_class(dataset_module.module_path							)
             lowercase__      :		List[DatasetBuilder]          =     []
             if self._beam_pipeline_options:
                    lowercase__      :		Any          =     beam.options.pipeline_options.PipelineOptions(
                        flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(','							) if opt]							)
             else:
                    lowercase__      :		Optional[int]          =     None
             if self._all_configs and len(builder_cls.BUILDER_CONFIGS							) > 0:
                    for builder_config in builder_cls.BUILDER_CONFIGS:
                           builders.append(
                               builder_cls(
                                   config_name=builder_config.name		,      data_dir=self._data_dir		,      hash=dataset_module.hash		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      )							)
             else:
                    builders.append(
                        builder_cls(
                            config_name=a		,      data_dir=self._data_dir		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      **self._config_kwargs		,      )							)

             for builder in builders:
                    builder.download_and_prepare(
                        download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                        if not self._force_redownload
                        else DownloadMode.FORCE_REDOWNLOAD		,      download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH							)		,      verification_mode=VerificationMode.NO_CHECKS
                        if self._ignore_verifications
                        else VerificationMode.ALL_CHECKS		,      try_from_hf_gcs=a		,      )
                    if self._save_infos:
                           builder._save_infos()

             print('Apache beam run successful.'							)

             # If save_infos=True, the dataset infos file is created next to the loaded module file.
             # Let's move it to the original directory of the dataset script, to allow the user to
             # upload them on S3 at the same time afterwards.
             if self._save_infos:
                    lowercase__      :		Dict          =     os.path.join(builder_cls.get_imported_module_dir()		,      config.DATASETDICT_INFOS_FILENAME							)

                    lowercase__      :		Union[str, Any]          =     Path(a							).name + '.py'

                    lowercase__      :		List[Any]          =     os.path.join(a		,      a							)
                    if os.path.isfile(a							):
                           lowercase__      :		Any          =     os.path.dirname(a							)
                    elif os.path.isfile(a							):
                           lowercase__      :		Optional[int]          =     path
                    else:  # in case of a remote dataset
                           print(f"""Dataset Infos file saved at {dataset_infos_path}"""							)
                           exit(1							)

                    # Move datasetinfo back to the user
                    lowercase__      :		List[Any]          =     os.path.join(a		,      config.DATASETDICT_INFOS_FILENAME							)
                    copyfile(a		,      a							)
                    print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""							)

