


'''simple docstring'''







import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


a__  : List[Any]				=	logging.getLogger(__name__)
@dataclass
class  lowercase_					:
    __UpperCAmelCase             =     field(
        metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}       )
    __UpperCAmelCase             =     field(
        default=a__  ,						metadata={'help': 'Pretrained config name or path if not the same as model_name'}       )
    __UpperCAmelCase             =     field(
        default=a__  ,						metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}       )
    __UpperCAmelCase             =     field(
        default=a__  ,						metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}  ,						)

@dataclass
class  lowercase_					:
    __UpperCAmelCase             =     field(
        metadata={'help': 'The name of the task to train selected in the list: ' + ', '.join(hans_processors.keys()       )}       )
    __UpperCAmelCase             =     field(
        metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}       )
    __UpperCAmelCase             =     field(
        default=128  ,						metadata={
            'help': (
                'The maximum total input sequence length after tokenization. Sequences longer '
                'than this will be truncated, sequences shorter will be padded.'
            )
        }  ,						)
    __UpperCAmelCase             =     field(
        default=a__  ,						metadata={'help': 'Overwrite the cached training and evaluation sets'}       )







def _UpperCamelCase	(					__A   )							->		Dict[str, torch.Tensor]:


      '''simple docstring'''



      UpperCamelCase__  =    default_data_collator(__A   )
      UpperCamelCase__  =    batch.pop("pairID"				,      __A   )
      return batch







def _UpperCamelCase	(					)							->		Optional[Any]:


      '''simple docstring'''



      UpperCamelCase__  =    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)   )
      UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__  =    parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir   )
          and os.listdir(training_args.output_dir   )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
                " --overwrite_output_dir to overcome."   )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"				,      datefmt="%m/%d/%Y %H:%M:%S"				,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN				,      )
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"				,      training_args.local_rank				,      training_args.device				,      training_args.n_gpu				,      bool(training_args.local_rank != -1   )				,      training_args.fpaa				,      )
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank   ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"				,      __A   )

      # Set seed
      set_seed(training_args.seed   )

      try:
            UpperCamelCase__  =    hans_tasks_num_labels[data_args.task_name]
      except KeyError:
            raise ValueError("Task not found: %s" % (data_args.task_name)   )

      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      UpperCamelCase__  =    AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path				,      num_labels=__A				,      finetuning_task=data_args.task_name				,      cache_dir=model_args.cache_dir				,      )
      UpperCamelCase__  =    AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path				,      cache_dir=model_args.cache_dir				,      )
      UpperCamelCase__  =    AutoModelForSequenceClassification.from_pretrained(
          model_args.model_name_or_path				,      from_tf=bool(".ckpt" in model_args.model_name_or_path   )				,      config=__A				,      cache_dir=model_args.cache_dir				,      )

      # Get datasets
      UpperCamelCase__  =    (
          HansDataset(
              data_dir=data_args.data_dir				,      tokenizer=__A				,      task=data_args.task_name				,      max_seq_length=data_args.max_seq_length				,      overwrite_cache=data_args.overwrite_cache				,      )
          if training_args.do_train
          else None
      )
      UpperCamelCase__  =    (
          HansDataset(
              data_dir=data_args.data_dir				,      tokenizer=__A				,      task=data_args.task_name				,      max_seq_length=data_args.max_seq_length				,      overwrite_cache=data_args.overwrite_cache				,      evaluate=__A				,      )
          if training_args.do_eval
          else None
      )

      # Initialize our Trainer
      UpperCamelCase__  =    Trainer(
          model=__A				,      args=__A				,      train_dataset=__A				,      eval_dataset=__A				,      data_collator=__A				,      )

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path   ) else None   )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir   )

    # Evaluation
      if training_args.do_eval:
            logger.info("*** Evaluate ***"   )

            UpperCamelCase__  =    trainer.predict(__A   )
            UpperCamelCase__  =    output.predictions
            UpperCamelCase__  =    np.argmax(__A				,      axis=1   )

            UpperCamelCase__  =    [ex.pairID for ex in eval_dataset]
            UpperCamelCase__  =    os.path.join(training_args.output_dir				,      "hans_predictions.txt"   )
            UpperCamelCase__  =    eval_dataset.get_labels()
            if trainer.is_world_master():
                  with open(__A				,      "w"   ) as writer:
                        writer.write("pairID,gold_label\n"   )
                        for pid, pred in zip(__A				,      __A   ):
                              writer.write("ex" + str(__A   ) + "," + label_list[int(__A   )] + "\n"   )

            trainer._log(output.metrics   )







def _UpperCamelCase	(					__A   )							->		Optional[Any]:


      '''simple docstring'''



      main()


if __name__ == "__main__":
  main()









'''simple docstring'''







import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
  import sqlitea

  import sqlalchemy


a__  : Optional[int]				=	datasets.utils.logging.get_logger(__name__)
@dataclass
class  lowercase_					(   datasets.BuilderConfig       ):
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     True
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     10000
    __UpperCAmelCase             =     None


    def        __a							(					self   ):
          if self.sql is None:
                raise ValueError("sql must be specified"   )
          if self.con is None:
                raise ValueError("con must be specified"   )


    def        __a							(					self  , a  , a = None  , ):
          UpperCamelCase__  =    config_kwargs.copy()
          # We need to stringify the Selectable object to make its hash deterministic

          # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
          UpperCamelCase__  =    config_kwargs["sql"]
          if not isinstance(a  , a   ):
                if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                      import sqlalchemy

                      if isinstance(a  , sqlalchemy.sql.Selectable   ):
                            UpperCamelCase__  =    sqlalchemy.create_engine(config_kwargs["con"].split("://"   )[0] + "://"   )
                            UpperCamelCase__  =    str(sql.compile(dialect=engine.dialect   )   )
                            UpperCamelCase__  =    sql_str
                      else:
                            raise TypeError(
                                f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(a   )}: {sql}'''   )
                else:
                      raise TypeError(
                          f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(a   )}: {sql}'''   )
          UpperCamelCase__  =    config_kwargs["con"]
          if not isinstance(a  , a   ):
                UpperCamelCase__  =    id(a   )
                logger.info(
                    f'''SQL connection \'con\' of type {type(a   )} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''   )

          return super().create_config_id(a  , custom_features=a   )


    @property
    def        __a							(					self   ):
          UpperCamelCase__  =    {
              "index_col": self.index_col,
              "columns": self.columns,
              "params": self.params,
              "coerce_float": self.coerce_float,
              "parse_dates": self.parse_dates,
          }
          return pd_read_sql_kwargs

class  lowercase_					(   datasets.ArrowBasedBuilder       ):
    __UpperCAmelCase             =     SqlConfig


    def        __a							(					self   ):
          return datasets.DatasetInfo(features=self.config.features   )


    def        __a							(					self  , a   ):
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN  , gen_kwargs={}   )]


    def        __a							(					self  , a   ):
          if self.config.features is not None:
                UpperCamelCase__  =    self.config.features.arrow_schema
                if all(not require_storage_cast(a   ) for feature in self.config.features.values()   ):
                      # cheaper cast
                      UpperCamelCase__  =    pa.Table.from_arrays([pa_table[field.name] for field in schema]  , schema=a   )
                else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      UpperCamelCase__  =    table_cast(a  , a   )
          return pa_table


    def        __a							(					self   ):
          UpperCamelCase__  =    self.config.chunksize
          UpperCamelCase__  =    pd.read_sql(
              self.config.sql  , self.config.con  , chunksize=a  , **self.config.pd_read_sql_kwargs   )
          UpperCamelCase__  =    [sql_reader] if chunksize is None else sql_reader
          for chunk_idx, df in enumerate(a   ):
                UpperCamelCase__  =    pa.Table.from_pandas(a   )
                yield chunk_idx, self._cast_table(a   )





