
'''simple docstring'''



import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
					import tensorflow as tf

					tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
					import torch

if is_datasets_available():
					from datasets import load_dataset


UpperCamelCase__ :						List[Any]    							=					5E-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors
def 				lowerCAmelCase_		(			SCREAMING_SNAKE_CASE_  )       ->       Any:




			"""simple docstring"""
			return PTtoTFCommand(
			    args.model_name      , args.local_dir      , args.max_error      , args.new_weights      , args.no_pr      , args.push      , args.extra_commit_description      , args.override_model_class      , )







class 				_a			(_UpperCAmelCase):





							"""simple docstring"""




							@staticmethod
							def     UpperCamelCase			(							A__ )      ->     Dict:
										_SCREAMING_SNAKE_CASE       	=			parser.add_parser(
										    """pt-to-tf"""    ,							help=(
										        """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
										        """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
										    )    ,							)
										train_parser.add_argument(
										    """--model-name"""    ,							type=lowerCAmelCase__    ,							required=lowerCAmelCase__    ,							help="""The model name, including owner/organization, as seen on the hub."""    ,							)
										train_parser.add_argument(
										    """--local-dir"""    ,							type=lowerCAmelCase__    ,							default=""""""    ,							help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""    ,							)
										train_parser.add_argument(
										    """--max-error"""    ,							type=lowerCAmelCase__    ,							default=lowerCAmelCase__    ,							help=(
										        F"Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."
										    )    ,							)
										train_parser.add_argument(
										    """--new-weights"""    ,							action="""store_true"""    ,							help="""Optional flag to create new TensorFlow weights, even if they already exist."""    ,							)
										train_parser.add_argument(
										    """--no-pr"""    ,							action="""store_true"""    ,							help="""Optional flag to NOT open a PR with converted weights.""" )
										train_parser.add_argument(
										    """--push"""    ,							action="""store_true"""    ,							help="""Optional flag to push the weights directly to `main` (requires permissions)"""    ,							)
										train_parser.add_argument(
										    """--extra-commit-description"""    ,							type=lowerCAmelCase__    ,							default=""""""    ,							help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""    ,							)
										train_parser.add_argument(
										    """--override-model-class"""    ,							type=lowerCAmelCase__    ,							default=lowerCAmelCase__    ,							help="""If you think you know better than the auto-detector, you can specify the model class here. """
										    """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""    ,							)
										train_parser.set_defaults(func=lowerCAmelCase__ )






							@staticmethod
							def     UpperCamelCase			(							A__    ,							A__ )      ->     Any:
										_SCREAMING_SNAKE_CASE       	=			set(pt_outputs.keys() )
										_SCREAMING_SNAKE_CASE       	=			set(tf_outputs.keys() )
										if pt_out_attrs != tf_out_attrs:
													raise ValueError(
													    F"The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:"
													    F" {tf_out_attrs})" )

										# 2. For each output attribute, computes the difference
										def _find_pt_tf_differences(A__    ,							A__    ,							A__    ,							A__="" ):
													# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
													# recursivelly, keeping the name of the attribute.
													if isinstance(lowerCAmelCase__    ,							torch.Tensor ):
																_SCREAMING_SNAKE_CASE       	=			np.max(np.abs(pt_out.numpy() - tf_out.numpy() ) )
																_SCREAMING_SNAKE_CASE       	=			tensor_difference
													else:
																_SCREAMING_SNAKE_CASE       	=			attr_name
																for i, pt_item in enumerate(lowerCAmelCase__ ):
																			# If it is a named attribute, we keep the name. Otherwise, just its index.
																			if isinstance(lowerCAmelCase__    ,							lowerCAmelCase__ ):
																						_SCREAMING_SNAKE_CASE       	=			root_name + pt_item
																						_SCREAMING_SNAKE_CASE       	=			tf_out[pt_item]
																						_SCREAMING_SNAKE_CASE       	=			pt_out[pt_item]
																			else:
																						_SCREAMING_SNAKE_CASE       	=			root_name + F"[{i}]"
																						_SCREAMING_SNAKE_CASE       	=			tf_out[i]
																			_SCREAMING_SNAKE_CASE       	=			_find_pt_tf_differences(lowerCAmelCase__    ,							lowerCAmelCase__    ,							lowerCAmelCase__    ,							lowerCAmelCase__ )

													return differences

										return _find_pt_tf_differences(lowerCAmelCase__    ,							lowerCAmelCase__    ,							{} )






							def __init__(							self    ,							A__    ,							A__    ,							A__    ,							A__    ,							A__    ,							A__    ,							A__    ,							A__    ,							*A__    ,							)      ->     str:
										_SCREAMING_SNAKE_CASE       	=			logging.get_logger("""transformers-cli/pt_to_tf""" )
										_SCREAMING_SNAKE_CASE       	=			model_name
										_SCREAMING_SNAKE_CASE       	=			local_dir if local_dir else os.path.join("""/tmp"""    ,							lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			max_error
										_SCREAMING_SNAKE_CASE       	=			new_weights
										_SCREAMING_SNAKE_CASE       	=			no_pr
										_SCREAMING_SNAKE_CASE       	=			push
										_SCREAMING_SNAKE_CASE       	=			extra_commit_description
										_SCREAMING_SNAKE_CASE       	=			override_model_class






							def     UpperCamelCase			(							self    ,							A__    ,							A__    ,							A__ )      ->     int:


										def _get_audio_input():
													_SCREAMING_SNAKE_CASE       	=			load_dataset("""hf-internal-testing/librispeech_asr_dummy"""    ,							"""clean"""    ,							split="""validation""" )
													_SCREAMING_SNAKE_CASE       	=			ds.sort("""id""" ).select(range(2 ) )[:2]["""audio"""]
													_SCREAMING_SNAKE_CASE       	=			[x["""array"""] for x in speech_samples]
													return raw_samples

										_SCREAMING_SNAKE_CASE       	=			type(pt_model.config )
										if model_config_class in PROCESSOR_MAPPING:
													_SCREAMING_SNAKE_CASE       	=			AutoProcessor.from_pretrained(self._local_dir )
													if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
																_SCREAMING_SNAKE_CASE       	=			processor.tokenizer.eos_token
										elif model_config_class in IMAGE_PROCESSOR_MAPPING:
													_SCREAMING_SNAKE_CASE       	=			AutoImageProcessor.from_pretrained(self._local_dir )
										elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
													_SCREAMING_SNAKE_CASE       	=			AutoFeatureExtractor.from_pretrained(self._local_dir )
										elif model_config_class in TOKENIZER_MAPPING:
													_SCREAMING_SNAKE_CASE       	=			AutoTokenizer.from_pretrained(self._local_dir )
													if processor.pad_token is None:
																_SCREAMING_SNAKE_CASE       	=			processor.eos_token
										else:
													raise ValueError(F"Unknown data processing type (model config type: {model_config_class})" )

										_SCREAMING_SNAKE_CASE       	=			set(inspect.signature(pt_model.forward ).parameters.keys() )
										_SCREAMING_SNAKE_CASE       	=			{}
										if "input_ids" in model_forward_signature:
													processor_inputs.update(
													    {
													        """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
													        """padding""": True,
													        """truncation""": True,
													    } )
										if "pixel_values" in model_forward_signature:
													_SCREAMING_SNAKE_CASE       	=			load_dataset("""cifar10"""    ,							"""plain_text"""    ,							split="""test""" )[:2]["""img"""]
													processor_inputs.update({"""images""": sample_images} )
										if "input_features" in model_forward_signature:
													_SCREAMING_SNAKE_CASE       	=			inspect.signature(processor.feature_extractor ).parameters
													# Pad to the largest input length by default but take feature extractor default
													# padding value if it exists e.g. "max_length" and is not False or None
													if "padding" in feature_extractor_signature:
																_SCREAMING_SNAKE_CASE       	=			feature_extractor_signature["""padding"""].default
																if default_strategy is not False and default_strategy is not None:
																			_SCREAMING_SNAKE_CASE       	=			default_strategy
																else:
																			_SCREAMING_SNAKE_CASE       	=			True
													else:
																_SCREAMING_SNAKE_CASE       	=			True
													processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy} )
										if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
													processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True} )
										_SCREAMING_SNAKE_CASE       	=			processor(**lowerCAmelCase__    ,							return_tensors="""pt""" )
										_SCREAMING_SNAKE_CASE       	=			processor(**lowerCAmelCase__    ,							return_tensors="""tf""" )

										# Extra input requirements, in addition to the input modality
										if (
										    config.is_encoder_decoder
										    or (hasattr(lowerCAmelCase__    ,							"""encoder""" ) and hasattr(lowerCAmelCase__    ,							"""decoder""" ))
										    or "decoder_input_ids" in tf_dummy_inputs
										):
													_SCREAMING_SNAKE_CASE       	=			np.asarray([[1], [1]]    ,							dtype=lowerCAmelCase__ ) * (pt_model.config.decoder_start_token_id or 0)
													pt_input.update({"""decoder_input_ids""": torch.tensor(lowerCAmelCase__ )} )
													tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(lowerCAmelCase__ )} )

										return pt_input, tf_input






							def     UpperCamelCase			(							self )      ->     Dict:


										if version.parse(huggingface_hub.__version__ ) < version.parse("""0.9.0""" ):
													raise ImportError(
													    """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
													    """ installation.""" )
										else:
													from huggingface_hub import Repository, create_commit
													from huggingface_hub._commit_api import CommitOperationAdd

										# Fetch remote data
										_SCREAMING_SNAKE_CASE       	=			Repository(local_dir=self._local_dir    ,							clone_from=self._model_name )

										# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
										_SCREAMING_SNAKE_CASE       	=			AutoConfig.from_pretrained(self._local_dir )
										_SCREAMING_SNAKE_CASE       	=			config.architectures
										if self._override_model_class is not None:
													if self._override_model_class.startswith("""TF""" ):
																_SCREAMING_SNAKE_CASE       	=			[self._override_model_class[2:]]
													else:
																_SCREAMING_SNAKE_CASE       	=			[self._override_model_class]
													try:
																_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							architectures[0] )
													except AttributeError:
																raise ValueError(F"Model class {self._override_model_class} not found in transformers." )
													try:
																_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							"""TF""" + architectures[0] )
													except AttributeError:
																raise ValueError(F"TF model class TF{self._override_model_class} not found in transformers." )
										elif architectures is None:  # No architecture defined -- use auto classes
													_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							"""AutoModel""" )
													_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							"""TFAutoModel""" )
													self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel""" )
										else:  # Architecture defined -- use it
													if len(lowerCAmelCase__ ) > 1:
																raise ValueError(F"More than one architecture was found, aborting. (architectures = {architectures})" )
													self._logger.warning(F"Detected architecture: {architectures[0]}" )
													_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							architectures[0] )
													try:
																_SCREAMING_SNAKE_CASE       	=			getattr(import_module("""transformers""" )    ,							"""TF""" + architectures[0] )
													except AttributeError:
																raise AttributeError(F"The TensorFlow equivalent of {architectures[0]} doesn't exist in transformers." )

        # Check the TF dummy inputs to see what keys we need in the forward pass
										_SCREAMING_SNAKE_CASE       	=			tf_class.from_config(lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			tf_from_pt_model.dummy_inputs

										del tf_from_pt_model  # Try to keep only one model in memory at a time

										# Load the model and get some basic inputs
										_SCREAMING_SNAKE_CASE       	=			pt_class.from_pretrained(self._local_dir )
										pt_model.eval()

										_SCREAMING_SNAKE_CASE	,							_SCREAMING_SNAKE_CASE       	=			self.get_inputs(lowerCAmelCase__    ,							lowerCAmelCase__    ,							lowerCAmelCase__ )

										with torch.no_grad():
													_SCREAMING_SNAKE_CASE       	=			pt_model(**lowerCAmelCase__    ,							output_hidden_states=lowerCAmelCase__ )
										del pt_model  # will no longer be used, and may have a large memory footprint

										_SCREAMING_SNAKE_CASE       	=			tf_class.from_pretrained(self._local_dir    ,							from_pt=lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			tf_from_pt_model(**lowerCAmelCase__    ,							output_hidden_states=lowerCAmelCase__    ,							training=lowerCAmelCase__ )

										# Confirms that cross loading PT weights into TF worked.
										_SCREAMING_SNAKE_CASE       	=			self.find_pt_tf_differences(lowerCAmelCase__    ,							lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			{k: v for k, v in crossload_differences.items() if """hidden""" not in k}
										_SCREAMING_SNAKE_CASE       	=			{k: v for k, v in crossload_differences.items() if """hidden""" in k}
										if len(lowerCAmelCase__ ) == 0 and architectures is not None:
													raise ValueError(
													    F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
													    """ output was found. All outputs start with \'hidden\'""" )
										_SCREAMING_SNAKE_CASE       	=			max(output_differences.values() ) if output_differences else 0.0
										_SCREAMING_SNAKE_CASE       	=			max(hidden_differences.values() )
										if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
													raise ValueError(
													    """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
													    + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
													    + """\n""".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error] )
													    + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
													    + """\n""".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error] ) )

										# Save the weights in a TF format (if needed) and confirms that the results are still good
										_SCREAMING_SNAKE_CASE       	=			os.path.join(self._local_dir    ,							lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			os.path.join(self._local_dir    ,							lowerCAmelCase__ )
										if (not os.path.exists(lowerCAmelCase__ ) and not os.path.exists(lowerCAmelCase__ )) or self._new_weights:
													tf_from_pt_model.save_pretrained(self._local_dir )
										del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

										_SCREAMING_SNAKE_CASE       	=			tf_class.from_pretrained(self._local_dir )
										_SCREAMING_SNAKE_CASE       	=			tf_model(**lowerCAmelCase__    ,							output_hidden_states=lowerCAmelCase__ )

										_SCREAMING_SNAKE_CASE       	=			self.find_pt_tf_differences(lowerCAmelCase__    ,							lowerCAmelCase__ )
										_SCREAMING_SNAKE_CASE       	=			{k: v for k, v in conversion_differences.items() if """hidden""" not in k}
										_SCREAMING_SNAKE_CASE       	=			{k: v for k, v in conversion_differences.items() if """hidden""" in k}
										if len(lowerCAmelCase__ ) == 0 and architectures is not None:
													raise ValueError(
													    F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
													    """ output was found. All outputs start with \'hidden\'""" )
										_SCREAMING_SNAKE_CASE       	=			max(output_differences.values() ) if output_differences else 0.0
										_SCREAMING_SNAKE_CASE       	=			max(hidden_differences.values() )
										if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
													raise ValueError(
													    """The converted TensorFlow model has different outputs, something went wrong!\n"""
													    + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
													    + """\n""".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error] )
													    + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
													    + """\n""".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error] ) )

										_SCREAMING_SNAKE_CASE       	=			"""Update TF weights""" if self._new_weights else """Add TF weights"""
										if self._push:
													repo.git_add(auto_lfs_track=lowerCAmelCase__ )
													repo.git_commit(lowerCAmelCase__ )
													repo.git_push(blocking=lowerCAmelCase__ )  # this prints a progress bar with the upload
													self._logger.warning(F"TF weights pushed into {self._model_name}" )
										elif not self._no_pr:
													self._logger.warning("""Uploading the weights into a new PR...""" )
													_SCREAMING_SNAKE_CASE       	=			(
													    """Model converted by the [`transformers`\' `pt_to_tf`"""
													    """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
													    """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
													    F"Maximum crossload output difference={max_crossload_output_diff:.3e}; "
													    F"Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n"
													    F"Maximum conversion output difference={max_conversion_output_diff:.3e}; "
													    F"Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n"
													)
													if self._max_error > MAX_ERROR:
																commit_descrition += (
																    F"\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!"
																)
													if self._extra_commit_description:
																commit_descrition += "\n\n" + self._extra_commit_description

													# sharded model -> adds all related files (index and .h5 shards)
													if os.path.exists(lowerCAmelCase__ ):
																_SCREAMING_SNAKE_CASE       	=			[
																    CommitOperationAdd(path_in_repo=lowerCAmelCase__    ,							path_or_fileobj=lowerCAmelCase__ )
																]
																for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5""" ):
																			operations += [
																			    CommitOperationAdd(path_in_repo=os.path.basename(lowerCAmelCase__ )    ,							path_or_fileobj=lowerCAmelCase__ )
																			]
													else:
																_SCREAMING_SNAKE_CASE       	=			[CommitOperationAdd(path_in_repo=lowerCAmelCase__    ,							path_or_fileobj=lowerCAmelCase__ )]

													_SCREAMING_SNAKE_CASE       	=			create_commit(
													    repo_id=self._model_name    ,							operations=lowerCAmelCase__    ,							commit_message=lowerCAmelCase__    ,							commit_description=lowerCAmelCase__    ,							repo_type="""model"""    ,							create_pr=lowerCAmelCase__    ,							).pr_url
													self._logger.warning(F"PR open in {hub_pr_url}" )








'''simple docstring'''



def 				lowerCAmelCase_		(			SCREAMING_SNAKE_CASE_  )       ->       str:




   """simple docstring"""
   return " ".join(sorted(set(sentence.split()  )  )  )


if __name__ == "__main__":
     import doctest

     doctest.testmod()






