


'''simple docstring'''







import argparse
import gc
import os
import sys
from pathlib import Path
from typing import List  # noqa: F401

import pytorch_lightning as pl
import torch
from finetune import SummarizationModule, TranslationModule
from finetune import main as ft_main
from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
from torch import nn

from transformers import AutoModelForSeqaSeqLM, MBartTokenizer, TaForConditionalGeneration
from transformers.models.bart.modeling_bart import shift_tokens_right
from utils import calculate_bleu, check_output_dir, freeze_params, label_smoothed_nll_loss, use_task_specific_params


# need the parent dir module
sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
from lightning_base import generic_train  # noqa
class  lowercase_					(   _a       ):
    __UpperCAmelCase             =     ["""loss""", """ce_loss""", """mlm_loss""", """hid_loss_enc""", """hid_loss_dec"""]


    def __init__(					self  , a   ):
          assert Path(hparams.data_dir   ).exists()
          UpperCamelCase__  =    Path(hparams.output_dir   )
          self.output_dir.mkdir(exist_ok=snake_case_   )

          UpperCamelCase__  =    self.output_dir.joinpath("student"   )

          UpperCamelCase__  =    str(snake_case_   )  # Tell lightning we are training the student
          UpperCamelCase__  =    AutoModelForSeqaSeqLM.from_pretrained(hparams.teacher   ).eval()
          use_task_specific_params(snake_case_  , hparams.task   )  # We copy good generation parameters to student by default
          if hparams.student is not None:
                UpperCamelCase__  =    AutoModelForSeqaSeqLM.from_pretrained(hparams.student   )
                use_task_specific_params(snake_case_  , hparams.task   )
                UpperCamelCase__  =    None, None
          else:
                UpperCamelCase__  =    create_student_by_copying_alternating_layers(
                    snake_case_  , e=hparams.student_encoder_layers  , d=hparams.student_decoder_layers  , save_path=snake_case_   )

          if hparams.length_penalty != -1:
                UpperCamelCase__  =    hparams.length_penalty
          UpperCamelCase__  =    hparams.teacher  # Use teacher's tokenizer
          super().__init__(snake_case_  , model=snake_case_  , config=student.config   )
          assert student.config.model_type == teacher.config.model_type, (
              f'''teacher, student model types should be the same, got {student.config.model_type} !='''
              f''' {teacher.config.model_type}'''
          )

          if student.config.model_type == "t5":
                UpperCamelCase__  =    len(student.get_encoder().block   )
                UpperCamelCase__  =    len(student.get_decoder().block   )
                UpperCamelCase__  =    len(teacher.get_encoder().block   )
                UpperCamelCase__  =    len(teacher.get_decoder().block   )
          else:
                UpperCamelCase__  =    student.config.encoder_layers
                UpperCamelCase__  =    student.config.decoder_layers
                UpperCamelCase__  =    teacher.config.encoder_layers
                UpperCamelCase__  =    teacher.config.decoder_layers

          UpperCamelCase__  =    not (hparams.student is None or hparams.teacher == hparams.student)
          UpperCamelCase__  =    (not self.different_base_models) and hparams.alpha_hid > 0
          UpperCamelCase__  =    self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
          # self.different_encoder determines whether we need to run the teacher encoder
          UpperCamelCase__  =    teacher
          freeze_params(self.teacher   )

          if not self.different_encoder:  # To save RAM, delete teacher encoder and freeze student encoder.
                try:
                      del self.teacher.model.encoder
                except AttributeError:  # T5
                      del self.teacher.encoder

          if e_layer_ids is None:
                UpperCamelCase__  =    list(range(snake_case_   )   )
          if d_layer_ids is None:
                UpperCamelCase__  =    list(range(snake_case_   )   )

          UpperCamelCase__  =    e_layer_ids, d_layer_ids  # type: List[int], List[int]

          if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
                if hparams.supervise_forward:
                      UpperCamelCase__  =    get_layers_to_supervise(
                          n_student=len(self.e_layer_ids   )  , n_teacher=snake_case_   )
                      UpperCamelCase__  =    get_layers_to_supervise(
                          n_student=len(self.d_layer_ids   )  , n_teacher=snake_case_   )
                else:  # student layer should emulate hidden states of the teacher layer it was copied from
                      UpperCamelCase__  =    self.e_layer_ids
                      UpperCamelCase__  =    self.d_layer_ids
          else:
                UpperCamelCase__  =    None
                UpperCamelCase__  =    None

          UpperCamelCase__  =    nn.KLDivLoss(reduction="batchmean"   )
          UpperCamelCase__  =    2.0
          UpperCamelCase__  =    hparams.alpha_mlm
          UpperCamelCase__  =    hparams.alpha_ce
          UpperCamelCase__  =    hparams.alpha_hid
          gc.collect()
          torch.cuda.empty_cache()


    def        __a							(					self  , a  , a  , a   ):
          UpperCamelCase__  =    mask[:, :, None].expand_as(snake_case_   )
          UpperCamelCase__  =    s_logits.size(-1   )
          UpperCamelCase__  =    torch.masked_select(snake_case_  , snake_case_   )  # (bs * seq_length * voc_size) modulo the 1s in mask
          UpperCamelCase__  =    torch.masked_select(snake_case_  , snake_case_   )  # (bs * seq_length * voc_size) modulo the 1s in mask
          UpperCamelCase__  =    s_logits_slct.view(-1  , snake_case_   )  # (bs * seq_length, voc_size) modulo the 1s in mask
          UpperCamelCase__  =    t_logits_slct.view(-1  , snake_case_   )  # (bs * seq_length, voc_size) modulo the 1s in mask
          assert t_logits_slct.size() == s_logits_slct.size()
          UpperCamelCase__  =    (
              self.ce_loss_fct(
                  nn.functional.log_softmax(s_logits_slct / self.temperature  , dim=-1   )  , nn.functional.softmax(t_logits_slct / self.temperature  , dim=-1   )  , )
              * (self.temperature) ** 2
          )
          return loss_ce


    @staticmethod
    def        __a							(					a  , a   ):
          SummarizationModule.add_model_specific_args(snake_case_  , snake_case_   )
          add_distill_args(snake_case_   )
          return parser


    def        __a							(					self  , a   ):
          UpperCamelCase__  =    self.tokenizer.pad_token_id
          UpperCamelCase__  =    batch["""input_ids"""], batch["""attention_mask"""], batch["""labels"""]
          if isinstance(self.model  , snake_case_   ):
                UpperCamelCase__  =    self.model._shift_right(snake_case_   )
          else:
                UpperCamelCase__  =    shift_tokens_right(snake_case_  , snake_case_   )

          # noinspection PyCallingNonCallable
          UpperCamelCase__  =    self(
              snake_case_  , attention_mask=snake_case_  , decoder_input_ids=snake_case_  , output_hidden_states=self.do_calc_hidden_loss  , output_attentions=snake_case_  , use_cache=snake_case_  , )
          UpperCamelCase__  =    student_outputs["""logits"""]

          # Same cross entropy vs. label smoothing logic as finetune.py
          assert lm_logits.shape[-1] == self.model.config.vocab_size
          if self.hparams.label_smoothing == 0:
                # Same behavior as modeling_bart.py, besides ignoring pad_token_id
                UpperCamelCase__  =    nn.CrossEntropyLoss(ignore_index=snake_case_   )
                UpperCamelCase__  =    loss_fct(lm_logits.view(-1  , lm_logits.shape[-1]   )  , labels.view(-1   )   )
          else:
                UpperCamelCase__  =    nn.functional.log_softmax(snake_case_  , dim=-1   )
                UpperCamelCase__  =    label_smoothed_nll_loss(
                    snake_case_  , snake_case_  , self.hparams.label_smoothing  , ignore_index=snake_case_   )

          def zero_tensor():
                return torch.tensor(0.0   ).type_as(snake_case_   )

          UpperCamelCase__  =    student_outputs[
              """encoder_last_hidden_state"""
          ]  # use this unless self.different_base_models
          UpperCamelCase__  =    zero_tensor(), zero_tensor()
          if self.different_encoder:  # compute encoder hidden state loss
                UpperCamelCase__  =    self.teacher.get_encoder()(
                    snake_case_  , attention_mask=snake_case_  , output_hidden_states=self.do_calc_hidden_loss  , )
                if self.different_base_models:
                      UpperCamelCase__  =    all_teacher_encoder_outputs["""last_hidden_state"""]
                elif self.do_calc_hidden_loss:
                      UpperCamelCase__  =    self.calc_hidden_loss(
                          snake_case_  , student_outputs["encoder_hidden_states"]  , all_teacher_encoder_outputs["hidden_states"]  , self.e_matches  , normalize_hidden=self.hparams.normalize_hidden  , )

          UpperCamelCase__  =    self.teacher(
              snake_case_  , attention_mask=snake_case_  , encoder_outputs=(teacher_enc_outputs,)  , decoder_input_ids=snake_case_  , output_hidden_states=self.do_calc_hidden_loss  , use_cache=snake_case_  , )
          UpperCamelCase__  =    decoder_input_ids.ne(snake_case_   )
          UpperCamelCase__  =    self.calc_ce_loss(snake_case_  , snake_case_  , teacher_outputs["logits"]   )
          if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
                UpperCamelCase__  =    self.calc_hidden_loss(
                    snake_case_  , student_outputs["decoder_hidden_states"]  , teacher_outputs["decoder_hidden_states"]  , self.d_matches  , normalize_hidden=self.hparams.normalize_hidden  , )

          UpperCamelCase__  =    (
              self.alpha_ce * loss_ce
              + self.alpha_mlm * student_lm_loss
              + self.hparams.alpha_hid * (hid_loss_enc + hid_loss_dec)
          )
          return blended_loss, loss_ce, student_lm_loss, hid_loss_enc, hid_loss_dec


    @staticmethod
    def        __a							(					a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    """expected list or tuple for hidden_states, got tensor of shape: """
          assert not isinstance(snake_case_  , torch.Tensor   ), f'''{msg}{hidden_states.shape}'''
          assert not isinstance(snake_case_  , torch.Tensor   ), f'''{msg}{hidden_states_T.shape}'''
          UpperCamelCase__  =    attention_mask.to(hidden_states[0]   )
          UpperCamelCase__  =    mask.sum() * hidden_states[0].size(-1   )
          UpperCamelCase__  =    torch.stack([hidden_states[i] for i in range(len(snake_case_   )   )]   )
          UpperCamelCase__  =    torch.stack([hidden_states_T[j] for j in matches]   )
          assert student_states.shape == teacher_states.shape, f'''{student_states.shape} != {teacher_states.shape}'''
          if normalize_hidden:
                UpperCamelCase__  =    nn.functional.layer_norm(snake_case_  , student_states.shape[1:]   )
                UpperCamelCase__  =    nn.functional.layer_norm(snake_case_  , teacher_states.shape[1:]   )
          UpperCamelCase__  =    nn.functional.mse_loss(snake_case_  , snake_case_  , reduction="none"   )
          UpperCamelCase__  =    (mse * mask.unsqueeze(0   ).unsqueeze(-1   )).sum() / valid_count
          return masked_mse







def _UpperCamelCase	(					__A   )							->		List[Any]:


      '''simple docstring'''



      parser.add_argument("--teacher"				,      type=_lowerCamelCase   )
      parser.add_argument("--alpha_ce"				,      default=0.8				,      type=_lowerCamelCase   )
      parser.add_argument("--alpha_mlm"				,      default=0.2				,      type=_lowerCamelCase   )
      parser.add_argument("--alpha_hid"				,      default=0.0				,      type=_lowerCamelCase				,      required=_lowerCamelCase   )
      parser.add_argument("--student"				,      type=_lowerCamelCase				,      required=_lowerCamelCase   )
      parser.add_argument("--student_decoder_layers"				,      default=12				,      type=_lowerCamelCase				,      required=_lowerCamelCase   )
      parser.add_argument("--student_encoder_layers"				,      default=12				,      type=_lowerCamelCase				,      required=_lowerCamelCase   )
      parser.add_argument("--no_teacher"				,      action="store_true"				,      default=_lowerCamelCase   )
      parser.add_argument("--length_penalty"				,      type=_lowerCamelCase				,      default=-1   )
      parser.add_argument("--supervise_forward"				,      action="store_true"				,      default=_lowerCamelCase   )
      parser.add_argument("--normalize_hidden"				,      action="store_true"				,      default=_lowerCamelCase   )
class  lowercase_					(   _a       ):
    __UpperCAmelCase             =     """translation"""
    __UpperCAmelCase             =     ["""bleu"""]
    __UpperCAmelCase             =     """bleu"""


    def __init__(					self  , a  , **a   ):
          super().__init__(snake_case_  , **snake_case_   )
          assert hparams.src_lang is not None
          assert hparams.tgt_lang is not None
          UpperCamelCase__  =    hparams.src_lang
          UpperCamelCase__  =    hparams.tgt_lang
          if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer  , snake_case_   ):
                UpperCamelCase__  =    self.tokenizer.lang_code_to_id[hparams.tgt_lang]


    def        __a							(					self  , a  , a   ):
          return calculate_bleu(snake_case_  , snake_case_   )


    @staticmethod
    def        __a							(					a  , a   ):
          TranslationModule.add_model_specific_args(snake_case_  , snake_case_   )
          add_distill_args(snake_case_   )
          return parser







def _UpperCamelCase	(					__A   )							->		Optional[Any]:


      '''simple docstring'''



      if args.no_teacher:
            UpperCamelCase__  =    TranslationModule if """translation""" in args.task else SummarizationModule
      else:  # DISTILL WITH TEACHER
            UpperCamelCase__  =    TranslationDistiller if """translation""" in args.task else SummarizationDistiller
      UpperCamelCase__  =    module_cls.__name__
      print(F'''using module {args.setup_cls}'''   )
      UpperCamelCase__  =    module_cls(_lowerCamelCase   )
      return model







def _UpperCamelCase	(					__A   )							->		str:


      '''simple docstring'''



      Path(args.output_dir   ).mkdir(exist_ok=_lowerCamelCase   )
      check_output_dir(_lowerCamelCase				,      expected_items=3   )

      UpperCamelCase__  =    create_module(_lowerCamelCase   )
      return ft_main(_lowerCamelCase				,      model=_lowerCamelCase   )


if __name__ == "__main__":
  a__  : Optional[int]				=	argparse.ArgumentParser()
  a__  : List[str]				=	pl.Trainer.add_argparse_args(parser)
  a__  : Union[str, Any]				=	SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
  a__  : Tuple				=	parser.parse_args()

  distill_main(args)





'''simple docstring'''







a__  : str				=	[
    ['ab-e1', 'ac-e3', 'ad-e5', 'bc-e4', 'bd-e2', 'be-e6', 'bh-e12', 'cd-e2', 'ce-e4',
     'de-e1', 'df-e8', 'dg-e5', 'dh-e10', 'ef-e3', 'eg-e2', 'fg-e6', 'gh-e6', 'hi-e3'],
    ['ab-e1', 'ac-e3', 'ad-e5', 'bc-e4', 'bd-e2', 'be-e6', 'cd-e2', 'de-e1', 'df-e8',
     'ef-e3', 'eg-e2', 'fg-e6'],
    ['ab-e1', 'ac-e3', 'bc-e4', 'bd-e2', 'de-e1', 'df-e8', 'dg-e5', 'ef-e3', 'eg-e2',
     'eh-e12', 'fg-e6', 'fh-e10', 'gh-e6'],
    ['ab-e1', 'ac-e3', 'bc-e4', 'bd-e2', 'bh-e12', 'cd-e2', 'df-e8', 'dh-e10'],
    ['ab-e1', 'ac-e3', 'ad-e5', 'bc-e4', 'bd-e2', 'cd-e2', 'ce-e4', 'de-e1', 'df-e8',
     'dg-e5', 'ef-e3', 'eg-e2', 'fg-e6']
]







def _UpperCamelCase	(					__A   )							->		Tuple:


      '''simple docstring'''



      UpperCamelCase__  =    set()
      for row in edge_array:
            for item in row:
                  distinct_edge.add(item[0]   )
      return list(__A   )







def _UpperCamelCase	(					__A				,      __A   )							->		int:


      '''simple docstring'''



      UpperCamelCase__  =    ["0"] * len(__A   )
      for i, row in enumerate(__A   ):
            for item in row:
                  if distinct_edge in item[0]:
                        UpperCamelCase__  =    "1"
                        break
      return "".join(__A   )







def _UpperCamelCase	(					__A   )							->		Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__  =    get_distinct_edge(__A   )
      UpperCamelCase__  =    {}

      for item in distinct_edge:
            UpperCamelCase__  =    get_bitcode(__A				,      __A   )
            # print('bit',bit)
            # bt=''.join(bit)
            UpperCamelCase__  =    bit.count("1"   )
            UpperCamelCase__  =    [s, bit]
      # Store [Distinct edge, WT(Bitcode), Bitcode] in descending order
      UpperCamelCase__  =    [
          [k, v[0], v[1]]
          for k, v in sorted(frequency_table.items()				,      key=lambda __A   : v[1][0]				,      reverse=__A   )
      ]
      return sorted_frequency_table







def _UpperCamelCase	(					__A   )							->		Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__  =    {}
      for _, item in enumerate(__A   ):
            nodes.setdefault(item[2]				,      []   ).append(item[0]   )
      return nodes







def _UpperCamelCase	(					__A   )							->		Tuple:


      '''simple docstring'''



      UpperCamelCase__  =    {}
      for key, value in nodes.items():
            UpperCamelCase__  =    value
      return cluster







def _UpperCamelCase	(					__A   )							->		str:


      '''simple docstring'''



      return [i * 100 / len(__A   ) for i in cluster]







def _UpperCamelCase	(					)							->		None:


      '''simple docstring'''



      print("\nNodes\n"   )
      for key, value in nodes.items():
            print(__A				,      __A   )
      print("\nSupport\n"   )
      print(__A   )
      print("\n Cluster \n"   )
      for key, value in sorted(cluster.items()				,      reverse=__A   ):
            print(__A				,      __A   )
      print("\n Graph\n"   )
      for key, value in graph.items():
            print(__A				,      __A   )
      print("\n Edge List of Frequent subgraphs \n"   )
      for edge_list in freq_subgraph_edge_list:
            print(__A   )







def _UpperCamelCase	(					__A				,      __A				,      __A				,      __A   )							->		Optional[int]:


      '''simple docstring'''



      for i in cluster[ca]:
            UpperCamelCase__  =    0
            UpperCamelCase__  =    ca + 1
            while ca < max(cluster.keys()   ):
                  for j in cluster[ca]:
                        if int(__A				,      2   ) & int(__A				,      2   ) == int(__A				,      2   ):
                              if tuple(nodes[i]   ) in graph:
                                    graph[tuple(nodes[i]   )].append(nodes[j]   )
                              else:
                                    UpperCamelCase__  =    [nodes[j]]
                              count += 1
                  if count == 0:
                        UpperCamelCase__  =    ca + 1
                  else:
                        break







def _UpperCamelCase	(					__A				,      __A   )							->		List[str]:


      '''simple docstring'''



      UpperCamelCase__  =    cluster[max(cluster.keys()   )]
      UpperCamelCase__  =    "Header"
      UpperCamelCase__  =    {}
      for i in x:
            if (["Header"],) in graph:
                  graph[(["Header"],)].append(x[i]   )
            else:
                  UpperCamelCase__  =    [x[i]]
      for i in x:
            UpperCamelCase__  =    [["Header"]]
      UpperCamelCase__  =    1
      while i < max(__A   ) - 1:
            create_edge(__A				,      __A				,      __A				,      __A   )
            UpperCamelCase__  =    i + 1
      return graph







def _UpperCamelCase	(					__A				,      __A				,      __A				,      __A=None   )							->		str:


      '''simple docstring'''



      UpperCamelCase__  =    (path or []) + [start]
      if start == end:
            paths.append(__A   )
      for node in graph[start]:
            if tuple(__A   ) not in path:
                  my_dfs(__A				,      tuple(__A   )				,      __A				,      __A   )







def _UpperCamelCase	(					__A				,      __A				,      __A   )							->		Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__  =    int(s / 100 * (len(__A   ) - 1)   )
      for i in cluster[k]:
            my_dfs(__A				,      tuple(cluster[k][i]   )				,      (["Header"],)   )







def _UpperCamelCase	(					__A   )							->		Optional[Any]:


      '''simple docstring'''



      UpperCamelCase__  =    []
      for edges in paths:
            UpperCamelCase__  =    []
            for j in range(len(__A   ) - 1   ):
                  UpperCamelCase__  =    list(edges[j]   )
                  for e in temp:
                        UpperCamelCase__  =    (e[0], e[1])
                        el.append(__A   )
            freq_sub_el.append(__A   )
      return freq_sub_el







def _UpperCamelCase	(					__A   )							->		Optional[int]:


      '''simple docstring'''



      for i in range(len(__A   )   ):
            for j in range(len(edge_array[i]   )   ):
                  UpperCamelCase__  =    edge_array[i][j].split("-"   )
                  UpperCamelCase__  =    t


if __name__ == "__main__":
  preprocess(edge_array)
  a__  : List[Any]				=	get_frequency_table(edge_array)
  a__  : Optional[int]				=	get_nodes(frequency_table)
  a__  : Any				=	get_cluster(nodes)
  a__  : Tuple				=	get_support(cluster)
  a__  : str				=	construct_graph(cluster, nodes)
  find_freq_subgraph_given_support(6_0, cluster, graph)
  a__  : list				=	[]
  a__  : str				=	freq_subgraphs_edge_list(paths)
  print_all()





