






from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


__UpperCamelCase   :						List[Any]							=  logging.get_logger(__name__)


class 					__SCREAMING_SNAKE_CASE(		nn.Module		):
							_UpperCAmelCase	      =    42
							_UpperCAmelCase	      =    jnp.floataa
							def lowerCAmelCase_	(       self: Dict  )						->						Tuple:
											snake_case__													=       self.config.text_config
											snake_case__													=       self.config.vision_config

											snake_case__													=       self.config.projection_dim
											snake_case__													=       text_config.hidden_size
											snake_case__													=       vision_config.hidden_size

											snake_case__													=       FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
											snake_case__													=       FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__    ,		_UpperCamelCase  ).module_class

											snake_case__													=       text_module(_UpperCamelCase    ,		dtype=self.dtype  )
											snake_case__													=       vision_module(_UpperCamelCase    ,		dtype=self.dtype  )

											snake_case__													=       nn.Dense(
											    self.projection_dim    ,		dtype=self.dtype    ,		kernel_init=jax.nn.initializers.normal(0.02  )    ,		use_bias=_UpperCamelCase    ,		)
											snake_case__													=       nn.Dense(
											    self.projection_dim    ,		dtype=self.dtype    ,		kernel_init=jax.nn.initializers.normal(0.02  )    ,		use_bias=_UpperCamelCase    ,		)
											snake_case__													=       self.param('logit_scale'    ,		jax.nn.initializers.ones    ,		[]  )


							def __call__(       self: Optional[int]    ,		UpperCamelCase: int=None    ,		UpperCamelCase: int=None    ,		UpperCamelCase: Optional[int]=None    ,		UpperCamelCase: Optional[Any]=None    ,		UpperCamelCase: Optional[int]=None    ,		UpperCamelCase: bool = True    ,		UpperCamelCase: str=None    ,		UpperCamelCase: Any=None    ,		UpperCamelCase: str=None    ,		)						->						str:
											snake_case__													=       return_dict if return_dict is not None else self.config.return_dict

											snake_case__													=       self.vision_model(
											    pixel_values=_UpperCamelCase    ,		deterministic=_UpperCamelCase    ,		output_attentions=_UpperCamelCase    ,		output_hidden_states=_UpperCamelCase    ,		return_dict=_UpperCamelCase    ,		)

											snake_case__													=       self.text_model(
											    input_ids=_UpperCamelCase    ,		attention_mask=_UpperCamelCase    ,		token_type_ids=_UpperCamelCase    ,		position_ids=_UpperCamelCase    ,		deterministic=_UpperCamelCase    ,		output_attentions=_UpperCamelCase    ,		output_hidden_states=_UpperCamelCase    ,		return_dict=_UpperCamelCase    ,		)

											snake_case__													=       vision_outputs[1]
											snake_case__													=       self.visual_projection(_UpperCamelCase  )

											snake_case__													=       text_outputs[1]
											snake_case__													=       self.text_projection(_UpperCamelCase  )

											# normalized features
											snake_case__													=       image_embeds / jnp.linalg.norm(_UpperCamelCase    ,		axis=-1    ,		keepdims=_UpperCamelCase  )
											snake_case__													=       text_embeds / jnp.linalg.norm(_UpperCamelCase    ,		axis=-1    ,		keepdims=_UpperCamelCase  )

											# cosine similarity as logits
											snake_case__													=       jnp.exp(self.logit_scale  )
											snake_case__													=       jnp.matmul(_UpperCamelCase    ,		image_embeds.T  ) * logit_scale
											snake_case__													=       logits_per_text.T

											if not return_dict:
															return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

											return FlaxCLIPOutput(
											    logits_per_image=_UpperCamelCase    ,		logits_per_text=_UpperCamelCase    ,		text_embeds=_UpperCamelCase    ,		image_embeds=_UpperCamelCase    ,		text_model_output=_UpperCamelCase    ,		vision_model_output=_UpperCamelCase    ,		)


class 					__SCREAMING_SNAKE_CASE(		__lowerCamelCase		):
							_UpperCAmelCase	      =    HybridCLIPConfig
							_UpperCAmelCase	      =    FlaxHybridCLIPModule
							def __init__(       self: str    ,		UpperCamelCase: HybridCLIPConfig    ,		UpperCamelCase: Optional[Tuple] = None    ,		UpperCamelCase: int = 0    ,		UpperCamelCase: jnp.dtype = jnp.floataa    ,		**UpperCamelCase: str    ,		)						->						Dict:





											if input_shape is None:
															snake_case__													=       ((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

											snake_case__													=       self.module_class(config=_UpperCamelCase    ,		dtype=_UpperCamelCase    ,		**_UpperCamelCase  )
											super().__init__(_UpperCamelCase    ,		_UpperCamelCase    ,		input_shape=_UpperCamelCase    ,		seed=_UpperCamelCase    ,		dtype=_UpperCamelCase  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: jax.random.PRNGKey    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: FrozenDict = None  )						->						FrozenDict:
											snake_case__													=       jnp.zeros(input_shape[0]    ,		dtype='i4'  )
											snake_case__													=       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_UpperCamelCase  ).shape[-1]  )    ,		input_shape[0]  )
											snake_case__													=       jnp.ones_like(_UpperCamelCase  )
											snake_case__													=       jnp.ones_like(_UpperCamelCase  )

											snake_case__													=       jax.random.normal(_UpperCamelCase    ,		input_shape[1]  )

											snake_case__							,  snake_case__													=       jax.random.split(_UpperCamelCase  )
											snake_case__													=       {'params': params_rng, 'dropout': dropout_rng}

											return self.module.init(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase  )["params"]
							def __call__(       self: Any    ,		UpperCamelCase: str    ,		UpperCamelCase: Any    ,		UpperCamelCase: List[str]=None    ,		UpperCamelCase: Any=None    ,		UpperCamelCase: Union[str, Any]=None    ,		UpperCamelCase: dict = None    ,		UpperCamelCase: jax.random.PRNGKey = None    ,		UpperCamelCase: bool = False    ,		UpperCamelCase: Optional[bool] = None    ,		UpperCamelCase: Optional[bool] = None    ,		UpperCamelCase: Optional[bool] = None    ,		)						->						List[Any]:
											snake_case__													=       output_attentions if output_attentions is not None else self.config.output_attentions
											snake_case__													=       (
											    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
											)
											snake_case__													=       return_dict if return_dict is not None else self.config.return_dict

											if position_ids is None:
															snake_case__													=       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_UpperCamelCase  ).shape[-1]  )    ,		input_ids.shape  )

											if token_type_ids is None:
															snake_case__													=       jnp.zeros_like(_UpperCamelCase  )

											if attention_mask is None:
															snake_case__													=       jnp.ones_like(_UpperCamelCase  )

											# Handle any PRNG if needed
											snake_case__													=       {}
											if dropout_rng is not None:
															snake_case__													=       dropout_rng

											return self.module.apply(
											    {'params': params or self.params}    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype=jnp.floataa  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		not train    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		rngs=_UpperCamelCase    ,		)
							def lowerCAmelCase_	(       self: Tuple    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: Any=None    ,		UpperCamelCase: Tuple=None    ,		UpperCamelCase: Tuple=None    ,		UpperCamelCase: dict = None    ,		UpperCamelCase: jax.random.PRNGKey = None    ,		UpperCamelCase: List[Any]=False    ,		)						->						Union[str, Any]:





											if position_ids is None:
															snake_case__													=       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_UpperCamelCase  ).shape[-1]  )    ,		input_ids.shape  )

											if token_type_ids is None:
															snake_case__													=       jnp.zeros_like(_UpperCamelCase  )

											if attention_mask is None:
															snake_case__													=       jnp.ones_like(_UpperCamelCase  )

											# Handle any PRNG if needed
											snake_case__													=       {}
											if dropout_rng is not None:
															snake_case__													=       dropout_rng

											def _get_features(UpperCamelCase: List[str]    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: Any    ,		UpperCamelCase: Dict    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: List[str]  ):
															snake_case__													=       module.text_model(
															    input_ids=_UpperCamelCase    ,		attention_mask=_UpperCamelCase    ,		position_ids=_UpperCamelCase    ,		token_type_ids=_UpperCamelCase    ,		deterministic=_UpperCamelCase    ,		)
															snake_case__													=       text_outputs[1]
															snake_case__													=       module.text_projection(_UpperCamelCase  )
															return text_features

											return self.module.apply(
											    {'params': params or self.params}    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		jnp.array(_UpperCamelCase    ,		dtype='i4'  )    ,		not train    ,		method=_get_features    ,		rngs=_UpperCamelCase    ,		)
							def lowerCAmelCase_	(       self: Union[str, Any]    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: dict = None    ,		UpperCamelCase: jax.random.PRNGKey = None    ,		UpperCamelCase: Union[str, Any]=False  )						->						List[str]:
											snake_case__													=       {}
											if dropout_rng is not None:
															snake_case__													=       dropout_rng

											def _get_features(UpperCamelCase: Optional[int]    ,		UpperCamelCase: Union[str, Any]    ,		UpperCamelCase: int  ):
															snake_case__													=       module.vision_model(pixel_values=_UpperCamelCase    ,		deterministic=_UpperCamelCase  )
															snake_case__													=       vision_outputs[1]  # pooled_output
															snake_case__													=       module.visual_projection(_UpperCamelCase  )
															return image_features

											return self.module.apply(
											    {'params': params or self.params}    ,		jnp.array(_UpperCamelCase    ,		dtype=jnp.floataa  )    ,		not train    ,		method=_get_features    ,		rngs=_UpperCamelCase    ,		)


							@classmethod
							def lowerCAmelCase_	(       cls: List[str]    ,		UpperCamelCase: str = None    ,		UpperCamelCase: str = None    ,		*UpperCamelCase: str    ,		**UpperCamelCase: Optional[Any]    ,		)						->						FlaxPreTrainedModel:
											snake_case__													=       {
											    argument[len('text_'  ) :]: value for argument, value in kwargs.items() if argument.startswith('text_'  )
											}

											snake_case__													=       {
											    argument[len('vision_'  ) :]: value for argument, value in kwargs.items() if argument.startswith('vision_'  )
											}

											# remove text, vision kwargs from kwargs
											for key in kwargs_text.keys():
															del kwargs["text_" + key]
											for key in kwargs_vision.keys():
															del kwargs["vision_" + key]

											# Load and initialize the text and vision model
											snake_case__													=       kwargs_text.pop('model'    ,		_UpperCamelCase  )
											if text_model is None:
															assert (
															    text_model_name_or_path is not None
															), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
															from transformers import FlaxAutoModel

															if "config" not in kwargs_text:
																			from transformers import AutoConfig

																			snake_case__													=       AutoConfig.from_pretrained(_UpperCamelCase  )
																			snake_case__													=       text_config

															snake_case__													=       FlaxAutoModel.from_pretrained(_UpperCamelCase    ,		*_UpperCamelCase    ,		**_UpperCamelCase  )

											snake_case__													=       kwargs_vision.pop('model'    ,		_UpperCamelCase  )
											if vision_model is None:
															assert (
															    vision_model_name_or_path is not None
															), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
															from transformers import FlaxAutoModel

															if "config" not in kwargs_vision:
																			from transformers import AutoConfig

																			snake_case__													=       AutoConfig.from_pretrained(_UpperCamelCase  )
																			snake_case__													=       vision_config

															snake_case__													=       FlaxAutoModel.from_pretrained(_UpperCamelCase    ,		*_UpperCamelCase    ,		**_UpperCamelCase  )

											# instantiate config with corresponding kwargs
											snake_case__													=       kwargs.pop('dtype'    ,		jnp.floataa  )
											snake_case__													=       HybridCLIPConfig.from_text_vision_configs(text_model.config    ,		vision_model.config    ,		**_UpperCamelCase  )

											# init model
											snake_case__													=       cls(_UpperCamelCase    ,		*_UpperCamelCase    ,		dtype=_UpperCamelCase    ,		**_UpperCamelCase  )

											if vision_config.model_type == "clip":
															snake_case__													=       vision_model.params['vision_model']
															snake_case__													=       vision_model.params['visual_projection']['kernel']
											else:
															snake_case__													=       vision_model.params

											snake_case__													=       text_model.params

											return model












import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
__UpperCamelCase   :						List[Any]							=  logging.getLogger(__name__)
def        a_	( )     ->     int:







				"""simple docstring"""
				snake_case__													=       argparse.ArgumentParser(description='PyTorch Transformer Language Model'					)
				parser.add_argument('--model_name' ,				type=_A ,				default='transfo-xl-wt103' ,				help='pretrained model name'					)
				parser.add_argument(
				    '--split' ,				type=_A ,				default='test' ,				choices=['all', 'valid', 'test'] ,				help='which split to evaluate'					)
				parser.add_argument('--batch_size' ,				type=_A ,				default=10 ,				help='batch size'					)
				parser.add_argument('--tgt_len' ,				type=_A ,				default=128 ,				help='number of tokens to predict'					)
				parser.add_argument('--ext_len' ,				type=_A ,				default=0 ,				help='length of the extended context'					)
				parser.add_argument('--mem_len' ,				type=_A ,				default=1600 ,				help='length of the retained previous heads'					)
				parser.add_argument('--clamp_len' ,				type=_A ,				default=1000 ,				help='max positional embedding index'					)
				parser.add_argument('--no_cuda' ,				action='store_true' ,				help='Do not use CUDA even though CUA is available'					)
				parser.add_argument('--work_dir' ,				type=_A ,				required=_A ,				help='path to the work_dir'					)
				parser.add_argument('--no_log' ,				action='store_true' ,				help='do not log the eval result'					)
				parser.add_argument('--same_length' ,				action='store_true' ,				help='set same length attention with masking'					)
				parser.add_argument('--server_ip' ,				type=_A ,				default='' ,				help='Can be used for distant debugging.'					)
				parser.add_argument('--server_port' ,				type=_A ,				default='' ,				help='Can be used for distant debugging.'					)
				snake_case__													=       parser.parse_args()
				assert args.ext_len >= 0, "extended context length must be non-negative"

				if args.server_ip and args.server_port:
								# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
								import ptvsd

								print('Waiting for debugger attach'					)
								ptvsd.enable_attach(address=(args.server_ip, args.server_port) ,				redirect_output=_A					)
								ptvsd.wait_for_attach()

				snake_case__													=       torch.device('cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu'					)
				logger.info('device: {}'.format(_A					)					)

				# Load a pre-processed dataset
				# You can also build the corpus yourself using TransfoXLCorpus methods
				# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
				# and tokenizing the dataset
				# The pre-processed corpus is a convertion (using the conversion script )
				snake_case__													=       TransfoXLCorpus.from_pretrained(args.model_name					)

				snake_case__													=       corpus.get_iterator('valid' ,				args.batch_size ,				args.tgt_len ,				device=_A ,				ext_len=args.ext_len					)
				snake_case__													=       corpus.get_iterator('test' ,				args.batch_size ,				args.tgt_len ,				device=_A ,				ext_len=args.ext_len					)

				# Load a pre-trained model
				snake_case__													=       TransfoXLLMHeadModel.from_pretrained(args.model_name					)
				model.to(_A					)

				logger.info(
				    'Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format(
				        args.batch_size ,				args.tgt_len ,				args.ext_len ,				args.mem_len ,				args.clamp_len					)					)

				model.reset_memory_length(args.mem_len					)
				if args.clamp_len > 0:
								snake_case__													=       args.clamp_len
				if args.same_length:
								snake_case__													=       True

				###############################################################################
				# Evaluation code
				###############################################################################
				def evaluate(_A					):
								# Turn on evaluation mode which disables dropout.
								model.eval()
								snake_case__							,  snake_case__													=       0, 0.0
								snake_case__													=       time.time()
								with torch.no_grad():
												snake_case__													=       None
												for idx, (data, target, seq_len) in enumerate(_A					):
																snake_case__													=       model(_A ,				lm_labels=_A ,				mems=_A					)
																snake_case__							,  snake_case__							,  snake_case__													=       ret
																snake_case__													=       loss.mean()
																total_loss += seq_len * loss.item()
																total_len += seq_len
												snake_case__													=       time.time() - start_time
								logger.info('Time : {:.2f}s, {:.2f}ms/segment'.format(_A ,				1000 * total_time / (idx + 1)					)					)
								return total_loss / total_len

				# Run on test data.
				if args.split == "all":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       evaluate(_A					)
				elif args.split == "valid":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       None
				elif args.split == "test":
								snake_case__													=       evaluate(_A					)
								snake_case__													=       None

				def format_log(_A ,				_A					):
								snake_case__													=       '| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format(_A ,				_A ,				math.exp(_A					)					)
								return log_str

				snake_case__													=       ''
				if valid_loss is not None:
								log_str += format_log(_A ,				'valid'					)
				if test_loss is not None:
								log_str += format_log(_A ,				'test'					)

				logger.info('=' * 100					)
				logger.info(_A					)
				logger.info('=' * 100					)


if __name__ == "__main__":
							main()




