



import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


A__     				=	logging.getLogger(__name__)

require_version("""pytorch_lightning>=1.0.4""")

A__     				=	{
    """base""": AutoModel,
    """sequence-classification""": AutoModelForSequenceClassification,
    """question-answering""": AutoModelForQuestionAnswering,
    """pretraining""": AutoModelForPreTraining,
    """token-classification""": AutoModelForTokenClassification,
    """language-modeling""": AutoModelWithLMHead,
    """summarization""": AutoModelForSeqaSeqLM,
    """translation""": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
A__     				=	{
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
A__     				=	sorted(arg_to_scheduler.keys())
A__     				=	"""{""" + """, """.join(arg_to_scheduler_choices) + """}"""





class  __lowerCAmelCase			(					pl.LightningModule	):




				def __init__(							self						,    _snake_case						,    _snake_case=None						,    _snake_case="base"						,    _snake_case=None						,    _snake_case=None						,    _snake_case=None						,    **_snake_case						,    ):



									"""simple docstring"""





									super().__init__()
									# TODO: move to self.save_hyperparameters()
									# self.save_hyperparameters()
									# can also expand arguments into trainer signature for easier reading

									self.save_hyperparameters(_snake_case		)
									_lowerCAmelCase      =   0
									_lowerCAmelCase      =   Path(self.hparams.output_dir		)
									_lowerCAmelCase      =   self.hparams.cache_dir if self.hparams.cache_dir else None
									if config is None:
														_lowerCAmelCase      =   AutoConfig.from_pretrained(
														    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path						,    **({"""num_labels""": num_labels} if num_labels is not None else {})						,    cache_dir=_snake_case						,    **_snake_case						,    )
									else:
														_lowerCAmelCase      =   config

									_lowerCAmelCase      =   ("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
									for p in extra_model_params:
														if getattr(self.hparams						,    _snake_case						,    _snake_case		):
																			assert hasattr(self.config						,    _snake_case		), F'model config doesn\'t have a `{p}` attribute'
																			setattr(self.config						,    _snake_case						,    getattr(self.hparams						,    _snake_case		)		)

									if tokenizer is None:
														_lowerCAmelCase      =   AutoTokenizer.from_pretrained(
														    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path						,    cache_dir=_snake_case						,    )
									else:
														_lowerCAmelCase      =   tokenizer
									_lowerCAmelCase      =   MODEL_MODES[mode]
									if model is None:
														_lowerCAmelCase      =   self.model_type.from_pretrained(
														    self.hparams.model_name_or_path						,    from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path		)						,    config=self.config						,    cache_dir=_snake_case						,    )
									else:
														_lowerCAmelCase      =   model




				def 							snake_case       (							self						,    *_snake_case						,    **_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self.model_type.from_pretrained(*_snake_case						,    **_snake_case		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   arg_to_scheduler[self.hparams.lr_scheduler]
									_lowerCAmelCase      =   get_schedule_func(
									    self.opt						,    num_warmup_steps=self.hparams.warmup_steps						,    num_training_steps=self.total_steps()		)
									_lowerCAmelCase      =   {"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
									return scheduler




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   self.model
									_lowerCAmelCase      =   ["""bias""", """LayerNorm.weight"""]
									_lowerCAmelCase      =   [
									    {
									        """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay		)],
									        """weight_decay""": self.hparams.weight_decay,
									    },
									    {
									        """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay		)],
									        """weight_decay""": 0.0,
									    },
									]
									if self.hparams.adafactor:
														_lowerCAmelCase      =   Adafactor(
														    _snake_case						,    lr=self.hparams.learning_rate						,    scale_parameter=_snake_case						,    relative_step=_snake_case		)

									else:
														_lowerCAmelCase      =   AdamW(
														    _snake_case						,    lr=self.hparams.learning_rate						,    eps=self.hparams.adam_epsilon		)
									_lowerCAmelCase      =   optimizer

									_lowerCAmelCase      =   self.get_lr_scheduler()

									return [optimizer], [scheduler]




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									return self.validation_step(_snake_case						,    _snake_case		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									return self.validation_end(_snake_case		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   max(1						,    self.hparams.gpus		)  # TODO: consider num_tpu_cores
									_lowerCAmelCase      =   self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
									return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									if mode == "test":
														_lowerCAmelCase      =   len(self.test_dataloader().dataset		)
									else:
														_lowerCAmelCase      =   self.get_dataloader("""train"""						,    self.hparams.train_batch_size						,    shuffle=_snake_case		)
														_lowerCAmelCase      =   len(self.train_dataloader().dataset		)




				def 							snake_case       (							self						,    _snake_case						,    _snake_case						,    _snake_case = False		):



									"""simple docstring"""





									raise NotImplementedError("""You must implement this for your task"""		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									return self.train_loader




				def 							snake_case       (							self		):



									"""simple docstring"""





									return self.get_dataloader("""dev"""						,    self.hparams.eval_batch_size						,    shuffle=_snake_case		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									return self.get_dataloader("""test"""						,    self.hparams.eval_batch_size						,    shuffle=_snake_case		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									return os.path.join(
									    self.hparams.data_dir						,    """cached_{}_{}_{}""".format(
									        _snake_case						,    list(filter(_snake_case						,    self.hparams.model_name_or_path.split("""/"""		)		)		).pop()						,    str(self.hparams.max_seq_length		)						,    )						,    )




				@pl.utilities.rank_zero_only
				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self.output_dir.joinpath("""best_tfmr"""		)
									_lowerCAmelCase      =   self.step_count
									self.model.save_pretrained(_snake_case		)
									self.tokenizer.save_pretrained(_snake_case		)




				@staticmethod
				def 							snake_case       (							_snake_case						,    _snake_case		):



									"""simple docstring"""





									parser.add_argument(
									    """--model_name_or_path"""						,    default=_snake_case						,    type=_snake_case						,    required=_snake_case						,    help="""Path to pretrained model or model identifier from huggingface.co/models"""						,    )
									parser.add_argument(
									    """--config_name"""						,    default=""""""						,    type=_snake_case						,    help="""Pretrained config name or path if not the same as model_name"""		)
									parser.add_argument(
									    """--tokenizer_name"""						,    default=_snake_case						,    type=_snake_case						,    help="""Pretrained tokenizer name or path if not the same as model_name"""						,    )
									parser.add_argument(
									    """--cache_dir"""						,    default=""""""						,    type=_snake_case						,    help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""						,    )
									parser.add_argument(
									    """--encoder_layerdrop"""						,    type=_snake_case						,    help="""Encoder layer dropout probability (Optional). Goes into model.config"""						,    )
									parser.add_argument(
									    """--decoder_layerdrop"""						,    type=_snake_case						,    help="""Decoder layer dropout probability (Optional). Goes into model.config"""						,    )
									parser.add_argument(
									    """--dropout"""						,    type=_snake_case						,    help="""Dropout probability (Optional). Goes into model.config"""						,    )
									parser.add_argument(
									    """--attention_dropout"""						,    type=_snake_case						,    help="""Attention dropout probability (Optional). Goes into model.config"""						,    )
									parser.add_argument("""--learning_rate"""						,    default=5e-5						,    type=_snake_case						,    help="""The initial learning rate for Adam."""		)
									parser.add_argument(
									    """--lr_scheduler"""						,    default="""linear"""						,    choices=_snake_case						,    metavar=_snake_case						,    type=_snake_case						,    help="""Learning rate scheduler"""						,    )
									parser.add_argument("""--weight_decay"""						,    default=0.0						,    type=_snake_case						,    help="""Weight decay if we apply some."""		)
									parser.add_argument("""--adam_epsilon"""						,    default=1e-8						,    type=_snake_case						,    help="""Epsilon for Adam optimizer."""		)
									parser.add_argument("""--warmup_steps"""						,    default=0						,    type=_snake_case						,    help="""Linear warmup over warmup_steps."""		)
									parser.add_argument("""--num_workers"""						,    default=4						,    type=_snake_case						,    help="""kwarg passed to DataLoader"""		)
									parser.add_argument("""--num_train_epochs"""						,    dest="""max_epochs"""						,    default=3						,    type=_snake_case		)
									parser.add_argument("""--train_batch_size"""						,    default=32						,    type=_snake_case		)
									parser.add_argument("""--eval_batch_size"""						,    default=32						,    type=_snake_case		)
									parser.add_argument("""--adafactor"""						,    action="""store_true"""		)





class  __lowerCAmelCase			(					pl.Callback	):




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   trainer.lr_schedulers[0]["""scheduler"""]
									_lowerCAmelCase      =   {F'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()		)}
									pl_module.logger.log_metrics(_snake_case		)




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									rank_zero_info("""***** Validation results *****"""		)
									_lowerCAmelCase      =   trainer.callback_metrics
									# Log results
									for key in sorted(_snake_case		):
														if key not in ["log", "progress_bar"]:
																			rank_zero_info("""{} = {}\n""".format(_snake_case						,    str(metrics[key]		)		)		)




				def 							snake_case       (							self						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									rank_zero_info("""***** Test results *****"""		)
									_lowerCAmelCase      =   trainer.callback_metrics
									# Log and save results to file
									_lowerCAmelCase      =   os.path.join(pl_module.hparams.output_dir						,    """test_results.txt"""		)
									with open(_snake_case						,    """w"""		) as writer:
														for key in sorted(_snake_case		):
																			if key not in ["log", "progress_bar"]:
																								rank_zero_info("""{} = {}\n""".format(_snake_case						,    str(metrics[key]		)		)		)
																								writer.write("""{} = {}\n""".format(_snake_case						,    str(metrics[key]		)		)		)


def 		_UpperCAmelCase    (				snake_case     ,						snake_case      ):





					"""simple docstring"""

					parser.add_argument(
					    """--output_dir"""     ,						default=snake_case     ,						type=snake_case     ,						required=snake_case     ,						help="""The output directory where the model predictions and checkpoints will be written."""     ,						)
					parser.add_argument(
					    """--fp16"""     ,						action="""store_true"""     ,						help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""     ,						)

					parser.add_argument(
					    """--fp16_opt_level"""     ,						type=snake_case     ,						default="""O2"""     ,						help=(
					        """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
					        """See details at https://nvidia.github.io/apex/amp.html"""
					    )     ,						)
					parser.add_argument("""--n_tpu_cores"""     ,						dest="""tpu_cores"""     ,						type=snake_case      )
					parser.add_argument("""--max_grad_norm"""     ,						dest="""gradient_clip_val"""     ,						default=1.0     ,						type=snake_case     ,						help="""Max gradient norm"""      )
					parser.add_argument("""--do_train"""     ,						action="""store_true"""     ,						help="""Whether to run training."""      )
					parser.add_argument("""--do_predict"""     ,						action="""store_true"""     ,						help="""Whether to run predictions on the test set."""      )
					parser.add_argument(
					    """--gradient_accumulation_steps"""     ,						dest="""accumulate_grad_batches"""     ,						type=snake_case     ,						default=1     ,						help="""Number of updates steps to accumulate before performing a backward/update pass."""     ,						)
					parser.add_argument("""--seed"""     ,						type=snake_case     ,						default=42     ,						help="""random seed for initialization"""      )
					parser.add_argument(
					    """--data_dir"""     ,						default=snake_case     ,						type=snake_case     ,						required=snake_case     ,						help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""     ,						)


def 		_UpperCAmelCase    (				snake_case     ,						snake_case     ,						snake_case=None     ,						snake_case=True     ,						snake_case=[]     ,						snake_case=None     ,						snake_case=None     ,						**snake_case     ,						):





					"""simple docstring"""

					pl.seed_everything(args.seed      )

					# init model
					_lowerCAmelCase      =   Path(model.hparams.output_dir      )
					odir.mkdir(exist_ok=snake_case      )

					# add custom checkpoints
					if checkpoint_callback is None:
										_lowerCAmelCase      =   pl.callbacks.ModelCheckpoint(
										    filepath=args.output_dir     ,						prefix="""checkpoint"""     ,						monitor="""val_loss"""     ,						mode="""min"""     ,						save_top_k=1      )
					if early_stopping_callback:
										extra_callbacks.append(snake_case      )
					if logging_callback is None:
										_lowerCAmelCase      =   LoggingCallback()

					_lowerCAmelCase      =   {}

					# TODO: remove with PyTorch 1.6 since pl uses native amp
					if args.fpaa:
										_lowerCAmelCase      =   16
										_lowerCAmelCase      =   args.fpaa_opt_level

					if args.gpus > 1:
										_lowerCAmelCase      =   """ddp"""

					_lowerCAmelCase      =   args.accumulate_grad_batches
					_lowerCAmelCase      =   extra_train_kwargs.get("""accelerator"""     ,						snake_case      )
					_lowerCAmelCase      =   extra_train_kwargs.get("""profiler"""     ,						snake_case      )

					_lowerCAmelCase      =   pl.Trainer.from_argparse_args(
					    snake_case     ,						weights_summary=snake_case     ,						callbacks=[logging_callback] + extra_callbacks     ,						logger=snake_case     ,						checkpoint_callback=snake_case     ,						**snake_case     ,						)

					if args.do_train:
										trainer.fit(snake_case      )

					return trainer








import json
import os
import subprocess
import sys
import warnings
from argparse import ArgumentParser
from contextlib import AbstractContextManager
from typing import Dict, List, Optional

import requests

from ..utils import logging
from . import BaseTransformersCLICommand


A__     				=	logging.get_logger(__name__)  # pylint: disable=invalid-name


A__     				=	"""lfs-multipart-upload"""





class  __lowerCAmelCase			(					lowerCamelCase__	):




				@staticmethod
				def 							snake_case       (							_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   parser.add_parser(
									    """lfs-enable-largefiles"""						,    help=(
									        """Deprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB."""
									    )						,    )
									enable_parser.add_argument("""path"""						,    type=_snake_case						,    help="""Local path to repository you want to configure."""		)
									enable_parser.set_defaults(func=lambda _snake_case		: LfsEnableCommand(_snake_case		)		)

									_lowerCAmelCase      =   parser.add_parser(
									    _snake_case						,    help=(
									        """Deprecated: use `huggingface-cli` instead. """
									        """Command will get called by git-lfs, do not call it directly."""
									    )						,    )
									upload_parser.set_defaults(func=lambda _snake_case		: LfsUploadCommand(_snake_case		)		)





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   args




				def 							snake_case       (							self		):



									"""simple docstring"""





									warnings.warn(
									    """Managing repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead."""		)
									_lowerCAmelCase      =   os.path.abspath(self.args.path		)
									if not os.path.isdir(_snake_case		):
														print("""This does not look like a valid git repo."""		)
														exit(1		)
									subprocess.run(
									    """git config lfs.customtransfer.multipart.path transformers-cli""".split()						,    check=_snake_case						,    cwd=_snake_case		)
									subprocess.run(
									    F'git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}'.split()						,    check=_snake_case						,    cwd=_snake_case						,    )
									print("""Local repo set up for largefiles"""		)


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					_lowerCAmelCase      =   json.dumps(snake_case      ) + """\n"""
					sys.stdout.write(snake_case      )
					sys.stdout.flush()


def 		_UpperCAmelCase    (				):





					"""simple docstring"""

					_lowerCAmelCase      =   json.loads(sys.stdin.readline().strip()      )

					if "terminate" in (msg.get("""type"""      ), msg.get("""event"""      )):
										# terminate message received
										return None

					if msg.get("""event"""      ) not in ("download", "upload"):
										logger.critical("""Received unexpected message"""      )
										sys.exit(1      )

					return msg





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   filepath
									_lowerCAmelCase      =   seek_from
									_lowerCAmelCase      =   read_limit
									_lowerCAmelCase      =   0




				def __enter__(							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   open(self.filepath						,    """rb"""		)
									self.f.seek(self.seek_from		)
									return self




				def __len__(							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   os.fstat(self.f.fileno()		).st_size
									return min(self.read_limit						,    total_length - self.seek_from		)




				def 							snake_case       (							self						,    _snake_case=-1		):



									"""simple docstring"""





									if self.n_seen >= self.read_limit:
														return b""
									_lowerCAmelCase      =   self.read_limit - self.n_seen
									_lowerCAmelCase      =   self.f.read(remaining_amount if n < 0 else min(_snake_case						,    _snake_case		)		)
									self.n_seen += len(_snake_case		)
									return data




				def __iter__(							self		):



									"""simple docstring"""





									yield self.read(n=4 * 1024 * 1024		)




				def __exit__(							self						,    *_snake_case		):



									"""simple docstring"""





									self.f.close()





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   args




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   json.loads(sys.stdin.readline().strip()		)
									if not (init_msg.get("""event"""		) == "init" and init_msg.get("""operation"""		) == "upload"):
														write_msg({"""error""": {"""code""": 32, """message""": """Wrong lfs init operation"""}}		)
														sys.exit(1		)

									# The transfer process should use the information it needs from the
									# initiation structure, and also perform any one-off setup tasks it
									# needs to do. It should then respond on stdout with a simple empty
									# confirmation structure, as follows:
									write_msg({}		)

									# After the initiation exchange, git-lfs will send any number of
									# transfer requests to the stdin of the transfer process, in a serial sequence.
									while True:
														_lowerCAmelCase      =   read_msg()
														if msg is None:
																			# When all transfers have been processed, git-lfs will send
																			# a terminate event to the stdin of the transfer process.
																			# On receiving this message the transfer process should
																			# clean up and terminate. No response is expected.
																			sys.exit(0		)

														_lowerCAmelCase      =   msg["""oid"""]
														_lowerCAmelCase      =   msg["""path"""]
														_lowerCAmelCase      =   msg["""action"""]["""href"""]
														_lowerCAmelCase      =   msg["""action"""]["""header"""]
														_lowerCAmelCase      =   int(header.pop("""chunk_size"""		)		)
														_lowerCAmelCase      =   list(header.values()		)

														_lowerCAmelCase      =   []
														for i, presigned_url in enumerate(_snake_case		):
																			with FileSlice(_snake_case						,    seek_from=i * chunk_size						,    read_limit=_snake_case		) as data:
																								_lowerCAmelCase      =   requests.put(_snake_case						,    data=_snake_case		)
																								r.raise_for_status()
																								parts.append(
																								    {
																								        """etag""": r.headers.get("""etag"""		),
																								        """partNumber""": i + 1,
																								    }		)
																								# In order to support progress reporting while data is uploading / downloading,
																								# the transfer process should post messages to stdout
																								write_msg(
																								    {
																								        """event""": """progress""",
																								        """oid""": oid,
																								        """bytesSoFar""": (i + 1) * chunk_size,
																								        """bytesSinceLast""": chunk_size,
																								    }		)
																								# Not precise but that's ok.

														_lowerCAmelCase      =   requests.post(
														    _snake_case						,    json={
														        """oid""": oid,
														        """parts""": parts,
														    }						,    )
														r.raise_for_status()

														write_msg({"""event""": """complete""", """oid""": oid}		)



