
# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


__lowerCAmelCase      :					Optional[Any]   	=       logging.get_logger(__name__)


if is_torch_available():
    import torch
    from torch import nn
    from torch.utils.data import Dataset

    from transformers import Trainer




    class 	UpperCAmelCase_      (			_A					):


      '''simple docstring'''







      def __init__(			self      :			Union[str, Any]      ,							UpperCamelCase__      :			int = 101   )      ->     Tuple:







        """simple docstring"""






        __magic_name__    					=	length
      def __len__(			self      :			Tuple   )      ->     Any:







        """simple docstring"""






        return self.length





      def __getitem__(			self      :			int      ,							UpperCamelCase__      :			Tuple   )      ->     int:







        """simple docstring"""






        return i







    class 	UpperCAmelCase_      :


      '''simple docstring'''







      def __call__(			self      :			str      ,							UpperCamelCase__      :			List[Any]   )      ->     List[str]:







        """simple docstring"""






        return {"input_ids": torch.tensor(UpperCamelCase__   ), "labels": torch.tensor(UpperCamelCase__   )}







    class 	UpperCAmelCase_      (			nn.Module					):


      '''simple docstring'''







      def __init__(			self      :			List[str]   )      ->     Union[str, Any]:







        """simple docstring"""






        super().__init__()
        # Add some (unused) params otherwise DDP will complain.
        __magic_name__    					=	nn.Linear(120      ,							80   )





      def 							_lowercase							(			self      :			Union[str, Any]      ,							UpperCamelCase__      :			Any      ,							UpperCamelCase__      :			List[Any]=None   )      ->     List[str]:







        """simple docstring"""






        if labels is not None:
          return torch.tensor(0.0      ,							device=input_ids.device   ), input_ids
        else:
          return input_ids

def a__   (	):






  '''simple docstring'''





  __magic_name__    					=	HfArgumentParser((TrainingArguments,) )
  sys.argv += ["--output_dir", "./examples"]
  __magic_name__    					=	parser.parse_args_into_dataclasses()[0]

  logger.warning(
      f'''Process rank: {training_args.local_rank}, device: {training_args.device}, '''
      f'''tpu_num_cores: {training_args.tpu_num_cores}''',					)

  # Essentially, what we want to verify in the distributed case is
  # that we get all samples back, in the right order.
  # (this is crucial for prediction for instance)
  for dataset_length in [1001, 256, 15]:
    __magic_name__    					=	DummyDataset(A_ )

    def compute_metrics(A_ ) -> Dict:
      __magic_name__    					=	list(range(len(A_ ) ) )
      __magic_name__    					=	p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
      return {"success": success}

    __magic_name__    					=	Trainer(
        model=DummyModel(),					args=A_,					data_collator=DummyDataCollator(),					eval_dataset=A_,					compute_metrics=A_,					)
    __magic_name__    					=	trainer.evaluate()
    logger.info(A_ )
    if metrics["eval_success"] is not True:
      logger.error(A_ )
      exit(1 )

    __magic_name__    					=	trainer.predict(A_ )
    logger.info(p.metrics )
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics )
      exit(1 )

    __magic_name__    					=	2

    __magic_name__    					=	trainer.evaluate()
    logger.info(A_ )
    if metrics["eval_success"] is not True:
      logger.error(A_ )
      exit(1 )

    __magic_name__    					=	trainer.predict(A_ )
    logger.info(p.metrics )
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics )
      exit(1 )

    __magic_name__    					=	None

  logger.info("""🔥 All distributed tests successful""" )

def a__   (	A_ ):






  '''simple docstring'''





  main()


if __name__ == "__main__":
    main()



import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
    import transformer_engine.pytorch as te

def a__   (	A_,					A_=True,					A_=True,					A_=True ):






  '''simple docstring'''





  if not is_fpa_available():
    raise ImportError("""Using `convert_model` requires transformer_engine to be installed.""" )
  for name, module in model.named_children():
    if isinstance(A_,					nn.Linear ) and to_transformer_engine and _convert_linear:
      # Return early if the linear layer weights are not multiples of 16
      if any(p % 16 != 0 for p in module.weight.shape ):
        return
      __magic_name__    					=	module.bias is not None
      __magic_name__    					=	te.Linear(
          module.in_features,					module.out_features,					bias=A_,					params_dtype=module.weight.dtype )
      __magic_name__    					=	module.weight.data.clone()
      if has_bias:
        __magic_name__    					=	module.bias.data.clone()

      setattr(A_,					A_,					A_ )
    elif isinstance(A_,					nn.LayerNorm ) and to_transformer_engine and _convert_ln:
      __magic_name__    					=	te.LayerNorm(module.normalized_shape[0],					eps=module.eps,					params_dtype=module.weight.dtype )
      __magic_name__    					=	module.weight.data.clone()
      __magic_name__    					=	module.bias.data.clone()

      setattr(A_,					A_,					A_ )
    elif isinstance(A_,					te.Linear ) and not to_transformer_engine and _convert_linear:
      __magic_name__    					=	module.bias is not None
      __magic_name__    					=	nn.Linear(
          module.in_features,					module.out_features,					bias=A_,					params_dtype=module.weight.dtype )
      __magic_name__    					=	module.weight.data.clone()
      if has_bias:
        __magic_name__    					=	module.bias.data.clone()

      setattr(A_,					A_,					A_ )
    elif isinstance(A_,					te.LayerNorm ) and not to_transformer_engine and _convert_ln:
      __magic_name__    					=	nn.LayerNorm(module.normalized_shape[0],					eps=module.eps,					params_dtype=module.weight.dtype )
      __magic_name__    					=	module.weight.data.clone()
      __magic_name__    					=	module.bias.data.clone()

      setattr(A_,					A_,					A_ )
    else:
      convert_model(
          A_,					to_transformer_engine=A_,					_convert_linear=_convert_linear,					_convert_ln=_convert_ln,					)





def a__   (	A_ ):






  '''simple docstring'''





  if not is_fpa_available():
    raise ImportError("""Using `has_transformer_engine_layers` requires transformer_engine to be installed.""" )
  for m in model.modules():
    if isinstance(A_,					(te.LayerNorm, te.Linear) ):
      return True
  return False

