import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import TvltImageProcessor
def 			UpperCamelCase__					(    A__      ,			A__=10      ,			A__=10      ,			A__=False      ,			A__=False   )       -> Optional[int]:
				snake_case__ :     List[str] 							=				[]
				for i in range(image_processor_tester.num_frames   ):
								video.append(np.random.randint(255      ,			size=(image_processor_tester.num_channels, width, height)      ,			dtype=np.uinta   )   )

				if not numpify and not torchify:
								# PIL expects the channel dimension as last dimension
								snake_case__ :     Optional[int] 							=				[Image.fromarray(np.moveaxis(A__      ,			0      ,			-1   )   ) for frame in video]

				if torchify:
								snake_case__ :     Any 							=				[torch.from_numpy(A__   ) for frame in video]

				return video
def 			UpperCamelCase__					(    A__      ,			A__=False      ,			A__=False      ,			A__=False   )       -> Optional[Any]:
				assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

				snake_case__ :     str 							=				[]
				for i in range(image_processor_tester.batch_size   ):
								if equal_resolution:
												snake_case__ :     Optional[int] 							=				image_processor_tester.max_resolution
								else:
												snake_case__ :     Any 							=				np.random.choice(
												    np.arange(image_processor_tester.min_resolution      ,			image_processor_tester.max_resolution   )      ,			2   )
												snake_case__ :     Any 							=				prepare_video(
												    image_processor_tester=A__      ,			width=A__      ,			height=A__      ,			numpify=A__      ,			torchify=A__      ,			)
								video_inputs.append(A__   )

				return video_inputs





class 			__snake_case  (      unittest.TestCase     ):




					def __init__(       self   ,	__UpperCamelCase   ,	__UpperCamelCase=7   ,	__UpperCamelCase=3   ,	__UpperCamelCase=4   ,	__UpperCamelCase=18   ,	__UpperCamelCase=30   ,	__UpperCamelCase=400   ,	__UpperCamelCase=True   ,	__UpperCamelCase=None   ,	__UpperCamelCase=True   ,	__UpperCamelCase=[0.5, 0.5, 0.5]   ,	__UpperCamelCase=[0.5, 0.5, 0.5]   ,	__UpperCamelCase=True   ,	__UpperCamelCase=None   ,	)					->  int:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				size if size is not None else {'shortest_edge': 18}
									snake_case__ :     str 							=				crop_size if crop_size is not None else {'height': 18, 'width': 18}

									snake_case__ :     List[str] 							=				parent
									snake_case__ :     int 							=				batch_size
									snake_case__ :     Any 							=				num_channels
									snake_case__ :     Tuple 							=				num_frames
									snake_case__ :     Optional[int] 							=				image_size
									snake_case__ :     List[str] 							=				min_resolution
									snake_case__ :     Union[str, Any] 							=				max_resolution
									snake_case__ :     List[Any] 							=				do_resize
									snake_case__ :     Dict 							=				size
									snake_case__ :     Optional[int] 							=				do_normalize
									snake_case__ :     Any 							=				image_mean
									snake_case__ :     Any 							=				image_std
									snake_case__ :     List[Any] 							=				do_center_crop
									snake_case__ :     int 							=				crop_size




					def       __a      (       self		)					->  Tuple:



									'''simple docstring'''







									return {
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_normalize": self.do_normalize,
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_center_crop": self.do_center_crop,
									    "crop_size": self.crop_size,
									}





@require_torch
@require_vision
class 			__snake_case  (      _lowerCamelCase   ,unittest.TestCase     ):
					__lowerCamelCase	 =       TvltImageProcessor if is_vision_available() else None




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				TvltImageProcessorTester(self		)




					@property
					def       __a      (       self		)					->  Optional[int]:



									'''simple docstring'''







									return self.image_processor_tester.prepare_image_processor_dict()




					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				self.image_processing_class(**self.image_processor_dict		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'image_mean'		)		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'image_std'		)		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'do_normalize'		)		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'do_resize'		)		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'do_center_crop'		)		)
									self.assertTrue(hasattr(__UpperCamelCase   ,	'size'		)		)




					def       __a      (       self		)					->  List[str]:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				self.image_processing_class(**self.image_processor_dict		)
									# create random PIL videos
									snake_case__ :     Optional[Any] 							=				prepare_video_inputs(self.image_processor_tester   ,	equal_resolution=__UpperCamelCase		)
									for video in video_inputs:
													self.assertIsInstance(__UpperCamelCase   ,	__UpperCamelCase		)
													self.assertIsInstance(video[0]   ,	Image.Image		)

									# Test not batched input
									snake_case__ :     Tuple 							=				image_processor(video_inputs[0]   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)

									# Test batched
									snake_case__ :     int 							=				image_processor(__UpperCamelCase   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)




					def       __a      (       self		)					->  List[Any]:



									'''simple docstring'''







									snake_case__ :     str 							=				self.image_processing_class(**self.image_processor_dict		)
									# create random numpy tensors
									snake_case__ :     Optional[Any] 							=				prepare_video_inputs(self.image_processor_tester   ,	equal_resolution=__UpperCamelCase   ,	numpify=__UpperCamelCase		)
									for video in video_inputs:
													self.assertIsInstance(__UpperCamelCase   ,	__UpperCamelCase		)
													self.assertIsInstance(video[0]   ,	np.ndarray		)

									# Test not batched input
									snake_case__ :     Dict 							=				image_processor(video_inputs[0]   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)

									# Test batched
									snake_case__ :     List[Any] 							=				image_processor(__UpperCamelCase   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)




					def       __a      (       self		)					->  int:



									'''simple docstring'''







									snake_case__ :     Any 							=				self.image_processing_class(**self.image_processor_dict		)
									# create random PyTorch tensors
									snake_case__ :     Optional[Any] 							=				prepare_video_inputs(self.image_processor_tester   ,	equal_resolution=__UpperCamelCase   ,	torchify=__UpperCamelCase		)
									for video in video_inputs:
													self.assertIsInstance(__UpperCamelCase   ,	__UpperCamelCase		)
													self.assertIsInstance(video[0]   ,	torch.Tensor		)

									# Test not batched input
									snake_case__ :     Any 							=				image_processor(video_inputs[0]   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        1,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)

									# Test batched
									snake_case__ :     List[Any] 							=				image_processor(__UpperCamelCase   ,	return_tensors='pt'		).pixel_values
									self.assertEqual(
									    encoded_videos.shape   ,	(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_frames,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['height'],
									        self.image_processor_tester.crop_size['width'],
									    )   ,	)



import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


lowerCAmelCase__					:       Tuple		=       logging.getLogger(__name__)





@dataclass
class 			__snake_case  :
					__lowerCamelCase	 =       field(
					    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}     )
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}     )
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}     )
					__lowerCamelCase	 =       field(default=_lowerCamelCase   ,metadata={"""help""": """Set this flag to use fast tokenization."""}     )
					# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
					# or just modify its tokenizer_config.json.
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}   ,)




def 			UpperCamelCase__						(    )       -> Optional[Any]:
				# See all possible arguments in src/transformers/training_args.py
				# or by passing the --help flag to this script.
				# We now keep distinct sets of args, for a cleaner separation of concerns.

				snake_case__ :     Optional[Any] 							=				HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)   )

				if len(sys.argv   ) == 2 and sys.argv[1].endswith('.json'   ):
								# If we pass only one argument to the script and it's the path to a json file,
								# let's parse it to get our arguments.
								snake_case__							,  snake_case__							,  snake_case__ :     Tuple 							=				parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]   )   )
				else:
								snake_case__							,  snake_case__							,  snake_case__ :     Optional[int] 							=				parser.parse_args_into_dataclasses()

				if (
				    os.path.exists(training_args.output_dir   )
				    and os.listdir(training_args.output_dir   )
				    and training_args.do_train
				    and not training_args.overwrite_output_dir
				):
								raise ValueError(
								    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
								    ' --overwrite_output_dir to overcome.'   )

				# Setup logging
				logging.basicConfig(
				    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'      ,			datefmt='%m/%d/%Y %H:%M:%S'      ,			level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN      ,			)
				logger.warning(
				    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'      ,			training_args.local_rank      ,			training_args.device      ,			training_args.n_gpu      ,			bool(training_args.local_rank != -1   )      ,			training_args.fpaa      ,			)
				# Set the verbosity to info of the Transformers logger (on main process only):
				if is_main_process(training_args.local_rank   ):
								transformers.utils.logging.set_verbosity_info()
								transformers.utils.logging.enable_default_handler()
								transformers.utils.logging.enable_explicit_format()
				logger.info('Training/evaluation parameters %s'      ,			A__   )

				# Prepare Question-Answering task
				# Load pretrained model and tokenizer
				#
				# Distributed training:
				# The .from_pretrained methods guarantee that only one local process can concurrently
				# download model & vocab.

				snake_case__ :     Optional[int] 							=				AutoConfig.from_pretrained(
				    model_args.config_name if model_args.config_name else model_args.model_name_or_path      ,			cache_dir=model_args.cache_dir      ,			)
				snake_case__ :     str 							=				AutoTokenizer.from_pretrained(
				    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path      ,			cache_dir=model_args.cache_dir      ,			use_fast=A__      ,			)
				snake_case__ :     Optional[int] 							=				AutoModelForQuestionAnswering.from_pretrained(
				    model_args.model_name_or_path      ,			from_tf=bool('.ckpt' in model_args.model_name_or_path   )      ,			config=A__      ,			cache_dir=model_args.cache_dir      ,			)

				# Get datasets
				snake_case__ :     Dict 							=				hasattr(model.config      ,			'lang2id'   )
				snake_case__ :     List[str] 							=				(
				    SquadDataset(
				        A__      ,			tokenizer=A__      ,			is_language_sensitive=A__      ,			cache_dir=model_args.cache_dir   )
				    if training_args.do_train
				    else None
				)
				snake_case__ :     str 							=				(
				    SquadDataset(
				        A__      ,			tokenizer=A__      ,			mode='dev'      ,			is_language_sensitive=A__      ,			cache_dir=model_args.cache_dir      ,			)
				    if training_args.do_eval
				    else None
				)

				# Data collator
				snake_case__ :     Dict 							=				DataCollatorWithPadding(A__      ,			pad_to_multiple_of=8   ) if training_args.fpaa else None

				# Initialize our Trainer
				snake_case__ :     str 							=				Trainer(
				    model=A__      ,			args=A__      ,			train_dataset=A__      ,			eval_dataset=A__      ,			data_collator=A__      ,			)

				# Training
				if training_args.do_train:
								trainer.train(
								    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path   ) else None   )
								trainer.save_model()
								# For convenience, we also re-save the tokenizer to the same directory,
								# so that you can share your model easily on huggingface.co/models =)
								if trainer.is_world_master():
												tokenizer.save_pretrained(training_args.output_dir   )
def 			UpperCamelCase__						(    A__   )       -> Tuple:
				# For xla_spawn (TPUs)
				main()


if __name__ == "__main__":
				main()


