




import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
    import torch

lowerCamelCase_  						=       logging.get_logger(__name__)



def 		UpperCAmelCase_   (     ):
    return importlib.util.find_spec("""deepspeed"""     ) is not None


if is_accelerate_available() and is_deepspeed_available():
    from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
    # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
    # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
    from builtins import object as DeepSpeedConfig
class 				__a (			_A					):


     """simple docstring"""


     def __init__(     self      :			Tuple    ,_UpperCamelCase      :			Optional[int]						)		->       Dict:


         '''simple docstring'''





         set_hf_deepspeed_config(self						)
         dep_version_check("""accelerate"""						)
         dep_version_check("""deepspeed"""						)
         super().__init__(UpperCamelCase__						)






class 				__a (			_A					):


     """simple docstring"""


     def __init__(     self      :			List[Any]    ,_UpperCamelCase      :			Dict						)		->       Any:


         '''simple docstring'''





         super().__init__(UpperCamelCase__						)
         SCREAMING_SNAKE_CASE__       =None
         SCREAMING_SNAKE_CASE__       =[]




     def   __A    (     self      :			Optional[Any]						)		->       Union[str, Any]:


         '''simple docstring'''





         if self._dtype is None:
             raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""						)
         return self._dtype




     def   __A    (     self      :			Any    ,_UpperCamelCase      :			Any						)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self.get_value(UpperCamelCase__						)
         if val is None:
             return False
         else:
             return val == "auto"




     def   __A    (     self      :			Optional[int]    ,_UpperCamelCase      :			List[Any]    ,_UpperCamelCase      :			List[str]    ,_UpperCamelCase      :			Optional[Any]=None    ,_UpperCamelCase      :			Tuple=True						)		->       Dict:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =self.find_config_node(UpperCamelCase__						)
         if config is None:
             return

         if config.get(UpperCamelCase__						) == "auto":
             SCREAMING_SNAKE_CASE__       =hf_val
             return

         if not must_match:
             return

         SCREAMING_SNAKE_CASE__       =config.get(UpperCamelCase__						)
         if ds_val is not None and ds_val != hf_val:
             self.mismatches.append(f"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""						)

     _A    :   List[str]     =  partialmethod(_A							,					must_match=_A					)




     def   __A    (     self      :			Any    ,_UpperCamelCase      :			Optional[Any]						)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
         self.fill_match(
             """train_micro_batch_size_per_gpu"""    ,args.per_device_train_batch_size    ,"""per_device_train_batch_size"""						)
         self.fill_match("""gradient_accumulation_steps"""    ,args.gradient_accumulation_steps    ,"""gradient_accumulation_steps"""						)
         self.fill_match("""train_batch_size"""    ,UpperCamelCase__    ,"""train_batch_size (calculated)"""						)
         self.fill_match("""gradient_clipping"""    ,args.max_grad_norm    ,"""max_grad_norm"""						)

         self.fill_match("""optimizer.params.lr"""    ,args.learning_rate    ,"""learning_rate"""						)
         self.fill_match("""optimizer.params.betas"""    ,[args.adam_betaa, args.adam_betaa]    ,"""adam_beta1+adam_beta2"""						)
         self.fill_match("""optimizer.params.eps"""    ,args.adam_epsilon    ,"""adam_epsilon"""						)
         self.fill_match("""optimizer.params.weight_decay"""    ,args.weight_decay    ,"""weight_decay"""						)

         self.fill_only("""scheduler.params.warmup_min_lr"""    ,0						)  # not a trainer arg
         self.fill_match("""scheduler.params.warmup_max_lr"""    ,args.learning_rate    ,"""learning_rate"""						)
         # total_num_steps - will get set in trainer_config_finalize

         # fp16
         if args.fpaa or args.fpaa_full_eval:
             SCREAMING_SNAKE_CASE__       ="""apex""" if args.fpaa_backend == """apex""" else """amp"""
         else:
             SCREAMING_SNAKE_CASE__       =None

         if args.save_on_each_node:
             # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
             SCREAMING_SNAKE_CASE__       =self.config.get("""checkpoint"""    ,{}						)
             SCREAMING_SNAKE_CASE__       =args.save_on_each_node

         # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
         # any here unless the user did the work
         self.fill_match(
             """fp16.enabled"""    ,((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")    ,"""fp16|fp16_full_eval+fp16_backend(amp)"""    ,)

         # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
         # ZeRO features
         self.fill_match("""amp.enabled"""    ,fpaa_backend == """apex"""    ,"""fp16+fp16_backend(apex)"""						)
         self.fill_match("""amp.opt_level"""    ,args.fpaa_opt_level    ,"""fp16_opt_level"""						)

         self.fill_match("""bf16.enabled"""    ,(args.bfaa or args.bfaa_full_eval)    ,"""bf16|bf16_full_eval"""						)

         # deepspeed's default mode is fp16 unless there is a config that says differently
         if self.is_true("""bf16.enabled"""						):
             SCREAMING_SNAKE_CASE__       =torch.bfloataa
         elif self.is_false("""fp16.enabled"""						):
             SCREAMING_SNAKE_CASE__       =torch.floataa
         else:
             SCREAMING_SNAKE_CASE__       =torch.floataa




     def   __A    (     self      :			List[str]    ,_UpperCamelCase      :			List[str]    ,_UpperCamelCase      :			Union[str, Any]    ,_UpperCamelCase      :			Tuple						)		->       Dict:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =[
             """zero_optimization.reduce_bucket_size""",
             """zero_optimization.stage3_prefetch_bucket_size""",
             """zero_optimization.stage3_param_persistence_threshold""",
         ]
         SCREAMING_SNAKE_CASE__       =[x for x in hidden_size_based_keys if self.is_auto(UpperCamelCase__						)]

         if len(UpperCamelCase__						) > 0:
             if hasattr(model.config    ,"""hidden_size"""						):
                 SCREAMING_SNAKE_CASE__       =model.config.hidden_size
             elif hasattr(model.config    ,"""hidden_sizes"""						):
                 # if there are many hidden sizes pick the largest one
                 SCREAMING_SNAKE_CASE__       =max(model.config.hidden_sizes						)
             else:
                 raise ValueError(
                     """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
                     """therefore it's not possible to automatically fill out the following `auto` entries """
                     f"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
                     """`auto` values for these keys with an integer value of your choice."""						)

             self.fill_only("""zero_optimization.reduce_bucket_size"""    ,hidden_size * hidden_size						)
             if self.is_zeroa():
                 # automatically assign the optimal config values based on model config
                 self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""    ,0.9 * hidden_size * hidden_size						)
                 self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""    ,1_0 * hidden_size						)

                 # scheduler
         self.fill_match("""scheduler.params.total_num_steps"""    ,UpperCamelCase__    ,"""num_training_steps (calculated)"""						)
         self.fill_match("""scheduler.params.warmup_num_steps"""    ,args.get_warmup_steps(UpperCamelCase__						)    ,"""warmup_steps"""						)

         if len(self.mismatches						) > 0:
             SCREAMING_SNAKE_CASE__       ="""\n""".join(self.mismatches						)
             raise ValueError(
                 """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
                 f""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'."""						)






lowerCamelCase_  						=       None



def 		UpperCAmelCase_   (     __UpperCamelCase     ):
    global _hf_deepspeed_config_weak_ref
    # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
    SCREAMING_SNAKE_CASE__       =weakref.ref(A_     )



def 		UpperCAmelCase_   (     ):
    global _hf_deepspeed_config_weak_ref
    SCREAMING_SNAKE_CASE__       =None



def 		UpperCAmelCase_   (     ):





    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().is_zeroa()
    else:
        return False



def 		UpperCAmelCase_   (     ):





    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().config
    else:
        return None



def 		UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase     ):
    from accelerate.utils import DummyOptim, DummyScheduler

    SCREAMING_SNAKE_CASE__       =hf_deepspeed_config.config

    # Optimizer + Scheduler
    # Currently supported combos:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Yes
    # 3. DS scheduler + HF optimizer: Yes
    # 4. HF scheduler + DS optimizer: No
    #
    # Unless Offload is enabled in which case it's:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Mostly*
    # 3. DS scheduler + HF optimizer: Mostly*
    # 4. HF scheduler + DS optimizer: No
    #
    # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

    SCREAMING_SNAKE_CASE__       =None
    if "optimizer" in config:
        if args.adafactor:
            raise ValueError(
                """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
                """Only one optimizer can be configured."""     )
        SCREAMING_SNAKE_CASE__       =DummyOptim(params=A_     )
    else:
        if hf_deepspeed_config.is_offload():
            logger.info(
                """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
                """ custom optimizer has both CPU and GPU implementation (except LAMB)"""     )

        # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
        # But trainer uses AdamW by default.
        SCREAMING_SNAKE_CASE__       =trainer.create_optimizer()
        # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
        SCREAMING_SNAKE_CASE__       =True

    SCREAMING_SNAKE_CASE__       =None
    if "scheduler" in config:
        SCREAMING_SNAKE_CASE__       =DummyScheduler(A_     )
    else:
        if isinstance(A_,    A_     ):
            raise ValueError(
                """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
                """Please configure a scheduler in the DeepSpeed config."""     )
        SCREAMING_SNAKE_CASE__       =trainer.create_scheduler(num_training_steps=A_,    optimizer=A_     )

    return optimizer, lr_scheduler



def 		UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase=False     ):
    from deepspeed.utils import logger as ds_logger

    SCREAMING_SNAKE_CASE__       =trainer.model
    SCREAMING_SNAKE_CASE__       =trainer.args

    SCREAMING_SNAKE_CASE__       =trainer.accelerator.state.deepspeed_plugin.hf_ds_config

    # resume config update - some bits like `model` and `num_training_steps` only become available during train
    hf_deepspeed_config.trainer_config_finalize(A_,    A_,    A_     )

    # set the Deepspeed log level consistent with the Trainer
    ds_logger.setLevel(args.get_process_log_level()     )

    if inference:
        # only Z3 makes sense for the inference
        if not hf_deepspeed_config.is_zeroa():
            raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"""     )

        # in case the training config is re-used for inference
        hf_deepspeed_config.del_config_sub_tree("""optimizer"""     )
        hf_deepspeed_config.del_config_sub_tree("""lr_scheduler"""     )
        SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =None, None
        SCREAMING_SNAKE_CASE__       =None
    else:
        SCREAMING_SNAKE_CASE__       =None  # important for when deepspeed_init is used as re-init
        SCREAMING_SNAKE_CASE__       =list(filter(lambda __UpperCamelCase     : p.requires_grad,    model.parameters()     )     )
        SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =deepspeed_optim_sched(
            A_,    A_,    A_,    A_,    A_     )

    # keep for quick debug:
    # from pprint import pprint; pprint(config)

    return optimizer, lr_scheduler




def 		UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase     ):
    import glob

    SCREAMING_SNAKE_CASE__       =sorted(glob.glob(f"""{checkpoint_path}/global_step*"""     )     )

    if len(A_     ) > 0:
        logger.info(f"""Attempting to resume from {checkpoint_path}"""     )
        # this magically updates self.optimizer and self.lr_scheduler
        SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =deepspeed_engine.load_checkpoint(
            A_,    load_optimizer_states=A_,    load_lr_scheduler_states=A_     )
        if load_path is None:
            raise ValueError(f"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""     )
    else:
        raise ValueError(f"""Can\'t find a valid checkpoint at {checkpoint_path}"""     )












import logging
import os
import sys
from dataclasses import dataclass, field
from typing import List, Optional

import torch
from datasets import Dataset
from torch import nn
from tqdm.auto import tqdm

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
    utils,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


lowerCamelCase_  						=       "\nDistills an NLI-based zero-shot classifier to a smaller, more efficient model with a fixed set of candidate class\nnames. Useful for speeding up zero-shot classification in cases where labeled training data is not available, but\nwhen only a single fixed set of classes is needed. Takes a teacher NLI model, student classifier model, unlabeled\ndataset, and set of K possible class names. Yields a single classifier with K outputs corresponding to the provided\nclass names.\n"

lowerCamelCase_  						=       logging.getLogger(__name__)
@dataclass
class 				__a :


     """simple docstring"""


     _A    :   Optional[str]     =  field(
         default="roberta-large-mnli"							,					metadata={"help": "The NLI/zero-shot teacher model to be distilled."}					)
     _A    :   Optional[str]     =  field(
         default="This example is {}."							,					metadata={
             "help": (
                 "Template used to turn class names into mock hypotheses for teacher NLI model. Must include {{}}"
                 "where class name is inserted."
             )
         }							,					)
     _A    :   Optional[int]     =  field(
         default=32							,					metadata={"help": "Batch size for generating teacher predictions."}					)
     _A    :   Optional[bool]     =  field(
         default=__lowerCamelCase							,					metadata={
             "help": (
                 "Allow multiple classes to be true rather than forcing them to sum to 1 (sometimes called"
                 "multi-class multi-label classification)."
             )
         }							,					)
     _A    :   Optional[float]     =  field(
         default=1.0							,					metadata={"help": "Temperature applied to teacher softmax for distillation."}					)






@dataclass
class 				__a :


     """simple docstring"""


     _A    :   Optional[str]     =  field(
         default="distilbert-base-uncased"							,					metadata={"help": "The NLI/zero-shot teacher model to be distilled."}					)






@dataclass
class 				__a :


     """simple docstring"""


     _A    :   str     =  field(metadata={"help": "Text file with one unlabeled instance per line."}					)
     _A    :   str     =  field(metadata={"help": "Text file with one class name per line."}					)
     _A    :   bool     =  field(
         default=__lowerCamelCase							,					metadata={"help": "Whether to use one of the fast tokenizer (backed by the Rust tokenizers library) or not."}							,					)






@dataclass
class 				__a (			__lowerCamelCase					):


     """simple docstring"""


     _A    :   Optional[str]     =  field(
         default=__lowerCamelCase							,					metadata={"help": "The output directory where the model predictions and checkpoints will be written."}							,					)
     _A    :   int     =  field(
         default=32							,					metadata={"help": "Batch size per GPU/TPU core/CPU for training."}					)
     _A    :   int     =  field(
         default=1_28							,					metadata={"help": "Batch size per GPU/TPU core/CPU for evaluation."}					)
     _A    :   float     =  field(default=1.0							,					metadata={"help": "Total number of training epochs to perform."}					)
     _A    :   bool     =  field(default=__lowerCamelCase							,					metadata={"help": "Whether to run training of student model."}					)
     _A    :   bool     =  field(
         default=__lowerCamelCase							,					metadata={
             "help": (
                 "Whether to evaluate the agreement of the final student predictions and the teacher predictions"
                 "after training."
             )
         }							,					)
     _A    :   Optional[int]     =  field(
         default=0							,					metadata={
             "help": (
                 "Limit the total amount of checkpoints."
                 "Deletes the older checkpoints in the output_dir. Default is 0 (no checkpoints)."
             )
         }							,					)






class 				__a (			__lowerCamelCase					):


     """simple docstring"""


     def   __A    (     self      :			int    ,_UpperCamelCase      :			Dict    ,_UpperCamelCase      :			Tuple    ,_UpperCamelCase      :			Optional[Any]=False						)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =inputs["""labels"""]
         SCREAMING_SNAKE_CASE__       =model(inputs["""input_ids"""]    ,attention_mask=inputs["""attention_mask"""]						)
         SCREAMING_SNAKE_CASE__       =outputs[0]

         SCREAMING_SNAKE_CASE__       =-torch.sum(target_p * logits.log_softmax(dim=-1						)    ,axis=-1						).mean()

         if return_outputs:
             return loss, outputs

         return loss






def      UpperCAmelCase_   (     __UpperCamelCase     ):
    SCREAMING_SNAKE_CASE__       =[]
    with open(__UpperCamelCase,    """r"""     ) as f:
        for line in f:
            SCREAMING_SNAKE_CASE__       =line.strip()
            if len(__UpperCamelCase     ) > 0:
                lines.append(__UpperCamelCase     )
    return lines



def      UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase     ):
    SCREAMING_SNAKE_CASE__       =[]
    SCREAMING_SNAKE_CASE__       =[]
    for example in examples:
        for name in class_names:
            premises.append(__UpperCamelCase     )
            hypotheses.append(hypothesis_template.format(__UpperCamelCase     )     )
    return premises, hypotheses



def      UpperCAmelCase_   (     __UpperCamelCase     ):
    for label, ind in config.labelaid.items():
        if label.lower().startswith("""entail"""     ):
            return ind
    logger.warning("""Could not identify entailment dimension from teacher config label2id. Setting to -1."""     )
    return -1



def      UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    ):
    SCREAMING_SNAKE_CASE__       =AutoModelForSequenceClassification.from_pretrained(__UpperCamelCase     )
    SCREAMING_SNAKE_CASE__       =model.config
    if not no_cuda and torch.cuda.is_available():
        SCREAMING_SNAKE_CASE__       =nn.DataParallel(model.cuda()     )
        batch_size *= len(model.device_ids     )
    SCREAMING_SNAKE_CASE__       =AutoTokenizer.from_pretrained(__UpperCamelCase,    use_fast=__UpperCamelCase     )

    SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =get_premise_hypothesis_pairs(__UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase     )
    SCREAMING_SNAKE_CASE__       =[]

    for i in tqdm(range(0,    len(__UpperCamelCase     ),    __UpperCamelCase     )     ):
        SCREAMING_SNAKE_CASE__       =premises[i : i + batch_size]
        SCREAMING_SNAKE_CASE__       =hypotheses[i : i + batch_size]

        SCREAMING_SNAKE_CASE__       =tokenizer(
            __UpperCamelCase,    __UpperCamelCase,    padding=__UpperCamelCase,    truncation="""only_first""",    return_tensors="""pt""",    )

        with torch.cuda.amp.autocast(enabled=__UpperCamelCase     ):
            with torch.no_grad():
                SCREAMING_SNAKE_CASE__       =model(**__UpperCamelCase     )
        logits.append(outputs.logits.detach().cpu().float()     )

    SCREAMING_SNAKE_CASE__       =get_entailment_id(__UpperCamelCase     )
    SCREAMING_SNAKE_CASE__       =-1 if entail_id == 0 else 0
    SCREAMING_SNAKE_CASE__       =torch.cat(__UpperCamelCase,    dim=0     )  # N*K x 3
    SCREAMING_SNAKE_CASE__       =logits.reshape(len(__UpperCamelCase     ),    len(__UpperCamelCase     ),    -1     )[..., [contr_id, entail_id]]  # N x K x 2

    if multi_label:
        # softmax over (contr, entail) logits for each class independently
        SCREAMING_SNAKE_CASE__       =(nli_logits / temperature).softmax(-1     )
    else:
        # softmax over entail logits across classes s.t. class probabilities sum to 1.
        SCREAMING_SNAKE_CASE__       =(nli_logits / temperature).softmax(1     )

    return nli_prob[..., 1]  # N x K



def      UpperCAmelCase_   (     ):
    SCREAMING_SNAKE_CASE__       =HfArgumentParser(
        (DataTrainingArguments, TeacherModelArguments, StudentModelArguments, DistillTrainingArguments),    description=__UpperCamelCase,    )

    if len(sys.argv     ) == 2 and sys.argv[1].endswith(""".json"""     ):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =parser.parse_json_file(
            json_file=os.path.abspath(sys.argv[1]     )     )
    else:
        SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__      ,		SCREAMING_SNAKE_CASE__       =parser.parse_args_into_dataclasses()

    # Detecting last checkpoint.
    SCREAMING_SNAKE_CASE__       =None
    if os.path.isdir(training_args.output_dir     ) and training_args.do_train and not training_args.overwrite_output_dir:
        SCREAMING_SNAKE_CASE__       =get_last_checkpoint(training_args.output_dir     )
        if last_checkpoint is None and len(os.listdir(training_args.output_dir     )     ) > 0:
            raise ValueError(
                f"""Output directory ({training_args.output_dir}) already exists and is not empty. """
                """Use --overwrite_output_dir to overcome."""     )
        elif last_checkpoint is not None:
            logger.info(
                f"""Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change """
                """the `--output_dir` or add `--overwrite_output_dir` to train from scratch."""     )

    # Setup logging
    logging.basicConfig(
        format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""",    datefmt="""%m/%d/%Y %H:%M:%S""",    handlers=[logging.StreamHandler(sys.stdout     )],    )
    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank     ) else logging.WARN     )

    # Log on each process the small summary:
    logger.warning(
        f"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"""
        + f"""distributed training: {bool(training_args.local_rank != -1     )}, 16-bits training: {training_args.fpaa}"""     )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank     ):
        utils.logging.set_verbosity_info()
        utils.logging.enable_default_handler()
        utils.logging.enable_explicit_format()

    if training_args.local_rank != -1:
        raise ValueError("""Distributed training is not currently supported."""     )
    if training_args.tpu_num_cores is not None:
        raise ValueError("""TPU acceleration is not currently supported."""     )

    logger.info(f"""Training/evaluation parameters {training_args}"""     )

    # Set seed before initializing model.
    set_seed(training_args.seed     )

    # 1. read in data
    SCREAMING_SNAKE_CASE__       =read_lines(data_args.data_file     )
    SCREAMING_SNAKE_CASE__       =read_lines(data_args.class_names_file     )

    # 2. get teacher predictions and load into dataset
    logger.info("""Generating predictions from zero-shot teacher model"""     )
    SCREAMING_SNAKE_CASE__       =get_teacher_predictions(
        teacher_args.teacher_name_or_path,    __UpperCamelCase,    __UpperCamelCase,    teacher_args.hypothesis_template,    teacher_args.teacher_batch_size,    teacher_args.temperature,    teacher_args.multi_label,    data_args.use_fast_tokenizer,    training_args.no_cuda,    training_args.fpaa,    )
    SCREAMING_SNAKE_CASE__       =Dataset.from_dict(
        {
            """text""": examples,
            """labels""": teacher_soft_preds,
        }     )

    # 3. create student
    logger.info("""Initializing student model"""     )
    SCREAMING_SNAKE_CASE__       =AutoModelForSequenceClassification.from_pretrained(
        student_args.student_name_or_path,    num_labels=len(__UpperCamelCase     )     )
    SCREAMING_SNAKE_CASE__       =AutoTokenizer.from_pretrained(student_args.student_name_or_path,    use_fast=data_args.use_fast_tokenizer     )
    SCREAMING_SNAKE_CASE__       =dict(enumerate(__UpperCamelCase     )     )
    SCREAMING_SNAKE_CASE__       ={label: i for i, label in enumerate(__UpperCamelCase     )}

    # 4. train student on teacher predictions
    SCREAMING_SNAKE_CASE__       =dataset.map(__UpperCamelCase,    input_columns="""text"""     )
    dataset.set_format("""torch"""     )

    def compute_metrics(__UpperCamelCase,    __UpperCamelCase=False     ):
        SCREAMING_SNAKE_CASE__       =p.predictions.argmax(-1     )
        SCREAMING_SNAKE_CASE__       =p.label_ids.argmax(-1     )  # "label_ids" are actually distributions
        return {"agreement": (preds == proxy_labels).mean().item()}

    SCREAMING_SNAKE_CASE__       =DistillationTrainer(
        model=__UpperCamelCase,    tokenizer=__UpperCamelCase,    args=__UpperCamelCase,    train_dataset=__UpperCamelCase,    compute_metrics=__UpperCamelCase,    )

    if training_args.do_train:
        logger.info("""Training student model on teacher predictions"""     )
        trainer.train()

    if training_args.do_eval:
        SCREAMING_SNAKE_CASE__       =trainer.evaluate(eval_dataset=__UpperCamelCase     )["""eval_agreement"""]
        logger.info(f"""Agreement of student and teacher predictions: {agreement * 100:0.2f}%"""     )

    trainer.save_model()


if __name__ == "__main__":
    main()






