

import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
_a			=       logging.getLogger(__name__)







def _a			(	)   ->   Tuple:

 """simple docstring"""






 __lowerCAmelCase:       Optional[Any]            =				argparse.ArgumentParser(description='PyTorch Transformer Language Model'		)
 parser.add_argument('--model_name'   , type=SCREAMING_SNAKE_CASE   , default='transfo-xl-wt103'   , help='pretrained model name'		)
 parser.add_argument(
     '--split'   , type=SCREAMING_SNAKE_CASE   , default='test'   , choices=['all', 'valid', 'test']   , help='which split to evaluate'		)
 parser.add_argument('--batch_size'   , type=SCREAMING_SNAKE_CASE   , default=10   , help='batch size'		)
 parser.add_argument('--tgt_len'   , type=SCREAMING_SNAKE_CASE   , default=1_28   , help='number of tokens to predict'		)
 parser.add_argument('--ext_len'   , type=SCREAMING_SNAKE_CASE   , default=0   , help='length of the extended context'		)
 parser.add_argument('--mem_len'   , type=SCREAMING_SNAKE_CASE   , default=16_00   , help='length of the retained previous heads'		)
 parser.add_argument('--clamp_len'   , type=SCREAMING_SNAKE_CASE   , default=10_00   , help='max positional embedding index'		)
 parser.add_argument('--no_cuda'   , action='store_true'   , help='Do not use CUDA even though CUA is available'		)
 parser.add_argument('--work_dir'   , type=SCREAMING_SNAKE_CASE   , required=SCREAMING_SNAKE_CASE   , help='path to the work_dir'		)
 parser.add_argument('--no_log'   , action='store_true'   , help='do not log the eval result'		)
 parser.add_argument('--same_length'   , action='store_true'   , help='set same length attention with masking'		)
 parser.add_argument('--server_ip'   , type=SCREAMING_SNAKE_CASE   , default=''   , help='Can be used for distant debugging.'		)
 parser.add_argument('--server_port'   , type=SCREAMING_SNAKE_CASE   , default=''   , help='Can be used for distant debugging.'		)
 __lowerCAmelCase:       int            =				parser.parse_args()
 assert args.ext_len >= 0, "extended context length must be non-negative"

 if args.server_ip and args.server_port:
  # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
  import ptvsd

  print('Waiting for debugger attach'		)
  ptvsd.enable_attach(address=(args.server_ip, args.server_port)   , redirect_output=SCREAMING_SNAKE_CASE		)
  ptvsd.wait_for_attach()

 __lowerCAmelCase:       Optional[Any]            =				torch.device('cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu'		)
 logger.info('device: {}'.format(SCREAMING_SNAKE_CASE		)		)

 # Load a pre-processed dataset
 # You can also build the corpus yourself using TransfoXLCorpus methods
 # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
 # and tokenizing the dataset
 # The pre-processed corpus is a convertion (using the conversion script )
 __lowerCAmelCase:       List[Any]            =				TransfoXLCorpus.from_pretrained(args.model_name		)

 __lowerCAmelCase:       str            =				corpus.get_iterator('valid'   , args.batch_size   , args.tgt_len   , device=SCREAMING_SNAKE_CASE   , ext_len=args.ext_len		)
 __lowerCAmelCase:       str            =				corpus.get_iterator('test'   , args.batch_size   , args.tgt_len   , device=SCREAMING_SNAKE_CASE   , ext_len=args.ext_len		)

 # Load a pre-trained model
 __lowerCAmelCase:       Tuple            =				TransfoXLLMHeadModel.from_pretrained(args.model_name		)
 model.to(SCREAMING_SNAKE_CASE		)

 logger.info(
     'Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format(
         args.batch_size   , args.tgt_len   , args.ext_len   , args.mem_len   , args.clamp_len		)		)

 model.reset_memory_length(args.mem_len		)
 if args.clamp_len > 0:
  __lowerCAmelCase:       List[Any]            =				args.clamp_len
 if args.same_length:
  __lowerCAmelCase:       List[Any]            =				True

 ###############################################################################
 # Evaluation code
 ###############################################################################
 def evaluate(SCREAMING_SNAKE_CASE		:      int		):
  # Turn on evaluation mode which disables dropout.
  model.eval()
  __lowerCAmelCase     ,							__lowerCAmelCase:       Optional[Any]            =				0, 0.0
  __lowerCAmelCase:       Any            =				time.time()
  with torch.no_grad():
   __lowerCAmelCase:       str            =				None
   for idx, (data, target, seq_len) in enumerate(SCREAMING_SNAKE_CASE		):
    __lowerCAmelCase:       Optional[int]            =				model(SCREAMING_SNAKE_CASE   , lm_labels=SCREAMING_SNAKE_CASE   , mems=SCREAMING_SNAKE_CASE		)
    __lowerCAmelCase     ,							__lowerCAmelCase     ,							__lowerCAmelCase:       List[Any]            =				ret
    __lowerCAmelCase:       str            =				loss.mean()
    total_loss += seq_len * loss.item()
    total_len += seq_len
   __lowerCAmelCase:       Union[str, Any]            =				time.time() - start_time
  logger.info('Time : {:.2f}s, {:.2f}ms/segment'.format(SCREAMING_SNAKE_CASE   , 10_00 * total_time / (idx + 1)		)		)
  return total_loss / total_len

 # Run on test data.
 if args.split == "all":
  __lowerCAmelCase:       Optional[int]            =				evaluate(SCREAMING_SNAKE_CASE		)
  __lowerCAmelCase:       List[str]            =				evaluate(SCREAMING_SNAKE_CASE		)
 elif args.split == "valid":
  __lowerCAmelCase:       List[Any]            =				evaluate(SCREAMING_SNAKE_CASE		)
  __lowerCAmelCase:       List[str]            =				None
 elif args.split == "test":
  __lowerCAmelCase:       int            =				evaluate(SCREAMING_SNAKE_CASE		)
  __lowerCAmelCase:       str            =				None

 def format_log(SCREAMING_SNAKE_CASE		:      Union[str, Any]   , SCREAMING_SNAKE_CASE		:      str		):
  __lowerCAmelCase:       Dict            =				'| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format(SCREAMING_SNAKE_CASE   , SCREAMING_SNAKE_CASE   , math.exp(SCREAMING_SNAKE_CASE		)		)
  return log_str

 __lowerCAmelCase:       int            =				''
 if valid_loss is not None:
  log_str += format_log(SCREAMING_SNAKE_CASE   , 'valid'		)
 if test_loss is not None:
  log_str += format_log(SCREAMING_SNAKE_CASE   , 'test'		)

 logger.info('=' * 1_00		)
 logger.info(SCREAMING_SNAKE_CASE		)
 logger.info('=' * 1_00		)


if __name__ == "__main__":
  main()








import inspect
import unittest

from transformers import is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import MODEL_MAPPING, PoolFormerConfig, PoolFormerForImageClassification, PoolFormerModel
  from transformers.models.poolformer.modeling_poolformer import POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
  from PIL import Image

  from transformers import PoolFormerImageProcessor






class 							A_			(		snake_case__ ):


     def   UpperCAmelCase  (							self      :		int )							->     Tuple:
      __lowerCAmelCase:       Optional[Any]            =				self.config_class(**self.inputs_dict )
      self.parent.assertTrue(hasattr(UpperCAmelCase      ,  'hidden_sizes' ) )
      self.parent.assertTrue(hasattr(UpperCAmelCase      ,  'num_encoder_blocks' ) )







class 							A_			:


     def __init__(							self      :		Optional[int]      ,  UpperCAmelCase      :		Any      ,  UpperCAmelCase      :		List[str]=1_3      ,  UpperCAmelCase      :		Optional[Any]=6_4      ,  UpperCAmelCase      :		Union[str, Any]=3      ,  UpperCAmelCase      :		Optional[Any]=4      ,  UpperCAmelCase      :		Any=[2, 2, 2, 2]      ,  UpperCAmelCase      :		Optional[Any]=[8, 4, 2, 1]      ,  UpperCAmelCase      :		str=[1_6, 3_2, 6_4, 1_2_8]      ,  UpperCAmelCase      :		str=[1, 4, 8, 1_6]      ,  UpperCAmelCase      :		Optional[Any]=False      ,  UpperCAmelCase      :		str=True      ,  UpperCAmelCase      :		Optional[int]="gelu"      ,  UpperCAmelCase      :		Union[str, Any]=0.1      ,  UpperCAmelCase      :		Optional[Any]=0.02      ,  UpperCAmelCase      :		List[Any]=3      ,  UpperCAmelCase      :		Optional[int]=None      ,  )							->     Any:
      __lowerCAmelCase:       List[str]            =				parent
      __lowerCAmelCase:       Union[str, Any]            =				batch_size
      __lowerCAmelCase:       Union[str, Any]            =				image_size
      __lowerCAmelCase:       str            =				num_channels
      __lowerCAmelCase:       Tuple            =				num_encoder_blocks
      __lowerCAmelCase:       Optional[int]            =				sr_ratios
      __lowerCAmelCase:       Union[str, Any]            =				depths
      __lowerCAmelCase:       int            =				hidden_sizes
      __lowerCAmelCase:       List[Any]            =				downsampling_rates
      __lowerCAmelCase:       List[Any]            =				is_training
      __lowerCAmelCase:       Union[str, Any]            =				use_labels
      __lowerCAmelCase:       Union[str, Any]            =				hidden_act
      __lowerCAmelCase:       Any            =				hidden_dropout_prob
      __lowerCAmelCase:       List[Any]            =				initializer_range
      __lowerCAmelCase:       Dict            =				num_labels
      __lowerCAmelCase:       List[Any]            =				scope


     def   UpperCAmelCase  (							self      :		Optional[int] )							->     List[str]:
      __lowerCAmelCase:       Dict            =				floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

      __lowerCAmelCase:       str            =				None
      if self.use_labels:
       __lowerCAmelCase:       int            =				ids_tensor([self.batch_size, self.image_size, self.image_size]      ,  self.num_labels )

      __lowerCAmelCase:       Dict            =				PoolFormerConfig(
          image_size=self.image_size      ,  num_channels=self.num_channels      ,  num_encoder_blocks=self.num_encoder_blocks      ,  depths=self.depths      ,  hidden_sizes=self.hidden_sizes      ,  hidden_act=self.hidden_act      ,  hidden_dropout_prob=self.hidden_dropout_prob      ,  initializer_range=self.initializer_range      ,  )

      return config, pixel_values, labels


     def   UpperCAmelCase  (							self      :		str      ,  UpperCAmelCase      :		Tuple      ,  UpperCAmelCase      :		int      ,  UpperCAmelCase      :		List[Any] )							->     str:
      __lowerCAmelCase:       Any            =				PoolFormerModel(config=UpperCAmelCase )
      model.to(UpperCAmelCase )
      model.eval()
      __lowerCAmelCase:       List[str]            =				model(UpperCAmelCase )
      __lowerCAmelCase:       Union[str, Any]            =				self.image_size // 32.0
      self.parent.assertEqual(
          result.last_hidden_state.shape      ,  (self.batch_size, self.hidden_sizes[-1], expected_height, expected_width) )


     def   UpperCAmelCase  (							self      :		Any )							->     int:
      __lowerCAmelCase:       List[str]            =				self.prepare_config_and_inputs()
      __lowerCAmelCase     ,							__lowerCAmelCase     ,							__lowerCAmelCase:       Union[str, Any]            =				config_and_inputs
      __lowerCAmelCase:       List[str]            =				{'pixel_values': pixel_values}
      return config, inputs_dict







@require_torch
class 							A_			(		snake_case__							,     snake_case__							,     unittest.TestCase ):
     _lowercase      :					Optional[Any]       =     (PoolFormerModel, PoolFormerForImageClassification) if is_torch_available() else ()
     _lowercase      :					List[Any]       =     (
         {'feature-extraction': PoolFormerModel, 'image-classification': PoolFormerForImageClassification}
         if is_torch_available()
         else {}
     )

     _lowercase      :					Tuple       =     False
     _lowercase      :					Tuple       =     False
     _lowercase      :					List[Any]       =     False
     _lowercase      :					Any       =     False
     _lowercase      :					List[Any]       =     False


     def   UpperCAmelCase  (							self      :		str )							->     Any:
      __lowerCAmelCase:       int            =				PoolFormerModelTester(self )
      __lowerCAmelCase:       int            =				PoolFormerConfigTester(self      ,  config_class=UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		Optional[Any] )							->     int:
      self.config_tester.run_common_tests()


     def   UpperCAmelCase  (							self      :		int )							->     int:
      __lowerCAmelCase:       Union[str, Any]            =				self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*UpperCAmelCase )


     @unittest.skip('PoolFormer does not use inputs_embeds' )
     def   UpperCAmelCase  (							self      :		Optional[int] )							->     Union[str, Any]:
      pass


     @unittest.skip('PoolFormer does not have get_input_embeddings method and get_output_embeddings methods' )
     def   UpperCAmelCase  (							self      :		int )							->     List[str]:
      pass


     def   UpperCAmelCase  (							self      :		List[str] )							->     List[str]:
      def check_hidden_states_output(UpperCAmelCase      :		List[str]      ,  UpperCAmelCase      :		int      ,  UpperCAmelCase      :		List[Any] ):
       __lowerCAmelCase:       Union[str, Any]            =				model_class(UpperCAmelCase )
       model.to(UpperCAmelCase )
       model.eval()

       with torch.no_grad():
        __lowerCAmelCase:       Dict            =				model(**self._prepare_for_class(UpperCAmelCase      ,  UpperCAmelCase ) )

       __lowerCAmelCase:       int            =				outputs.hidden_states

       __lowerCAmelCase:       Optional[int]            =				self.model_tester.num_encoder_blocks
       self.assertEqual(len(UpperCAmelCase )      ,  UpperCAmelCase )

       # verify the first hidden states (first block)
       self.assertListEqual(
           list(hidden_states[0].shape[-3:] )      ,  [
               self.model_tester.hidden_sizes[0],
               self.model_tester.image_size // 4,
               self.model_tester.image_size // 4,
           ]      ,  )

      __lowerCAmelCase     ,							__lowerCAmelCase:       Dict            =				self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       __lowerCAmelCase:       Optional[int]            =				True
       check_hidden_states_output(UpperCAmelCase      ,  UpperCAmelCase      ,  UpperCAmelCase )

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       __lowerCAmelCase:       Optional[Any]            =				True

       check_hidden_states_output(UpperCAmelCase      ,  UpperCAmelCase      ,  UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		List[Any] )							->     Optional[int]:
      if not self.model_tester.is_training:
       return

      __lowerCAmelCase     ,							__lowerCAmelCase:       Union[str, Any]            =				self.model_tester.prepare_config_and_inputs_for_common()
      __lowerCAmelCase:       Optional[int]            =				True

      for model_class in self.all_model_classes:
       if model_class in get_values(UpperCAmelCase ):
        continue
       __lowerCAmelCase:       str            =				model_class(UpperCAmelCase )
       model.to(UpperCAmelCase )
       model.train()
       __lowerCAmelCase:       Any            =				self._prepare_for_class(UpperCAmelCase      ,  UpperCAmelCase      ,  return_labels=UpperCAmelCase )
       __lowerCAmelCase:       str            =				model(**UpperCAmelCase ).loss
       loss.backward()


     def   UpperCAmelCase  (							self      :		Dict )							->     Tuple:
      __lowerCAmelCase     ,							__lowerCAmelCase:       Union[str, Any]            =				self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       __lowerCAmelCase:       Union[str, Any]            =				model_class(UpperCAmelCase )
       __lowerCAmelCase:       Dict            =				inspect.signature(model.forward )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       __lowerCAmelCase:       List[str]            =				[*signature.parameters.keys()]

       __lowerCAmelCase:       List[Any]            =				['pixel_values']
       self.assertListEqual(arg_names[:1]      ,  UpperCAmelCase )


     @slow
     def   UpperCAmelCase  (							self      :		int )							->     str:
      for model_name in POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       __lowerCAmelCase:       List[str]            =				PoolFormerModel.from_pretrained(UpperCAmelCase )
       self.assertIsNotNone(UpperCAmelCase )







def _a			(	)   ->   Any:

 """simple docstring"""






 __lowerCAmelCase:       Optional[Any]            =				Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'		)
 return image






@require_torch
class 							A_			(		unittest.TestCase ):


     @slow
     def   UpperCAmelCase  (							self      :		Optional[Any] )							->     int:
      __lowerCAmelCase:       Optional[int]            =				PoolFormerImageProcessor()
      __lowerCAmelCase:       Any            =				PoolFormerForImageClassification.from_pretrained('sail/poolformer_s12' ).to(UpperCAmelCase )

      __lowerCAmelCase:       int            =				image_processor(images=prepare_img()      ,  return_tensors='pt' ).to(UpperCAmelCase )

      # forward pass
      with torch.no_grad():
       __lowerCAmelCase:       List[Any]            =				model(**UpperCAmelCase )

      # verify the logits
      __lowerCAmelCase:       List[Any]            =				torch.Size((1, 1_0_0_0) )
      self.assertEqual(outputs.logits.shape      ,  UpperCAmelCase )

      __lowerCAmelCase:       Union[str, Any]            =				torch.tensor([-0.6113, 0.1685, -0.0492] ).to(UpperCAmelCase )
      self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,  UpperCAmelCase      ,  atol=1E-4 ) )





