




"""simple docstring"""




import json
import os
import subprocess
import sys
import warnings
from argparse import ArgumentParser
from contextlib import AbstractContextManager
from typing import Dict, List, Optional

import requests

from ..utils import logging
from . import BaseTransformersCLICommand


_lowerCAmelCase  :				Dict						=				logging.get_logger(__name__)  # pylint: disable=invalid-name


_lowerCAmelCase  :				int						=				"lfs-multipart-upload"
class      _UpperCamelCase (			lowerCAmelCase						):
      @staticmethod
      def 		UpperCAmelCase_  (     lowerCamelCase				:ArgumentParser )       ->     Optional[int]:
       UpperCAmelCase__								=  parser.add_parser(
           "lfs-enable-largefiles"	, help=(
               "Deprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB."
           )	, )
       enable_parser.add_argument("path"	, type=lowerCamelCase	, help="Local path to repository you want to configure." )
       enable_parser.set_defaults(func=lambda lowerCamelCase : LfsEnableCommand(lowerCamelCase ) )

       UpperCAmelCase__								=  parser.add_parser(
           lowerCamelCase	, help=(
               "Deprecated: use `huggingface-cli` instead. "
               "Command will get called by git-lfs, do not call it directly."
           )	, )
       upload_parser.set_defaults(func=lambda lowerCamelCase : LfsUploadCommand(lowerCamelCase ) )



class      _UpperCamelCase :
      def __init__(     self				:Union[str, Any]	, lowerCamelCase				:List[str] )       ->     Tuple:
       UpperCAmelCase__								=  args







      def 		UpperCAmelCase_  (     self				:List[Any] )       ->     List[Any]:
       warnings.warn(
           "Managing repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead." )
       UpperCAmelCase__								=  os.path.abspath(self.args.path )
       if not os.path.isdir(lowerCamelCase ):
        print("This does not look like a valid git repo." )
        exit(1 )
       subprocess.run(
           "git config lfs.customtransfer.multipart.path transformers-cli".split()	, check=lowerCamelCase	, cwd=lowerCamelCase )
       subprocess.run(
           f'''git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}'''.split()	, check=lowerCamelCase	, cwd=lowerCamelCase	, )
       print("Local repo set up for largefiles" )



def   lowerCAmelCase						(						_lowerCAmelCase							:				Dict		):




 """simple docstring"""







 UpperCAmelCase__								=  json.dumps(_lowerCAmelCase		) + "\n"
 sys.stdout.write(_lowerCAmelCase		)
 sys.stdout.flush()



def   lowerCAmelCase						(						):




 """simple docstring"""







 UpperCAmelCase__								=  json.loads(sys.stdin.readline().strip()		)

 if "terminate" in (msg.get("type"		), msg.get("event"		)):
  # terminate message received
  return None

 if msg.get("event"		) not in ("download", "upload"):
  logger.critical("Received unexpected message"		)
  sys.exit(1		)

 return msg
class      _UpperCamelCase (			lowerCAmelCase						):
      def __init__(     self				:List[str]	, lowerCamelCase				:str	, lowerCamelCase				:int	, lowerCamelCase				:int )       ->     Optional[int]:
       UpperCAmelCase__								=  filepath
       UpperCAmelCase__								=  seek_from
       UpperCAmelCase__								=  read_limit
       UpperCAmelCase__								=  0
      def __enter__(     self				:Optional[int] )       ->     Dict:
       UpperCAmelCase__								=  open(self.filepath	, "rb" )
       self.f.seek(self.seek_from )
       return self
      def __len__(     self				:List[str] )       ->     Tuple:
       UpperCAmelCase__								=  os.fstat(self.f.fileno() ).st_size
       return min(self.read_limit	, total_length - self.seek_from )
      def 		UpperCAmelCase_  (     self				:str	, lowerCamelCase				:List[str]=-1 )       ->     int:
       if self.n_seen >= self.read_limit:
        return b""
       UpperCAmelCase__								=  self.read_limit - self.n_seen
       UpperCAmelCase__								=  self.f.read(remaining_amount if n < 0 else min(lowerCamelCase	, lowerCamelCase ) )
       self.n_seen += len(lowerCamelCase )
       return data
      def __iter__(     self				:Optional[Any] )       ->     Tuple:
       yield self.read(n=4 * 1024 * 1024 )







      def __exit__(     self				:Union[str, Any]	, *lowerCamelCase				:str )       ->     List[str]:
       self.f.close()



class      _UpperCamelCase :
      def __init__(     self				:Dict	, lowerCamelCase				:Tuple )       ->     Tuple:
       UpperCAmelCase__								=  args







      def 		UpperCAmelCase_  (     self				:int )       ->     Optional[int]:
       # Immediately after invoking a custom transfer process, git-lfs
       # sends initiation data to the process over stdin.
       # This tells the process useful information about the configuration.
       UpperCAmelCase__								=  json.loads(sys.stdin.readline().strip() )
       if not (init_msg.get("event" ) == "init" and init_msg.get("operation" ) == "upload"):
        write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}} )
        sys.exit(1 )

       # The transfer process should use the information it needs from the
       # initiation structure, and also perform any one-off setup tasks it
       # needs to do. It should then respond on stdout with a simple empty
       # confirmation structure, as follows:
       write_msg({} )

       # After the initiation exchange, git-lfs will send any number of
       # transfer requests to the stdin of the transfer process, in a serial sequence.
       while True:
        UpperCAmelCase__								=  read_msg()
        if msg is None:
         # When all transfers have been processed, git-lfs will send
         # a terminate event to the stdin of the transfer process.
         # On receiving this message the transfer process should
         # clean up and terminate. No response is expected.
         sys.exit(0 )

        UpperCAmelCase__								=  msg["oid"]
        UpperCAmelCase__								=  msg["path"]
        UpperCAmelCase__								=  msg["action"]["href"]
        UpperCAmelCase__								=  msg["action"]["header"]
        UpperCAmelCase__								=  int(header.pop("chunk_size" ) )
        UpperCAmelCase__								=  list(header.values() )

        UpperCAmelCase__								=  []
        for i, presigned_url in enumerate(lowerCamelCase ):
         with FileSlice(lowerCamelCase	, seek_from=i * chunk_size	, read_limit=lowerCamelCase ) as data:
          UpperCAmelCase__								=  requests.put(lowerCamelCase	, data=lowerCamelCase )
          r.raise_for_status()
          parts.append(
              {
                  "etag": r.headers.get("etag" ),
                  "partNumber": i + 1,
              } )
          # In order to support progress reporting while data is uploading / downloading,
          # the transfer process should post messages to stdout
          write_msg(
              {
                  "event": "progress",
                  "oid": oid,
                  "bytesSoFar": (i + 1) * chunk_size,
                  "bytesSinceLast": chunk_size,
              } )
          # Not precise but that's ok.

        UpperCAmelCase__								=  requests.post(
            lowerCamelCase	, json={
                "oid": oid,
                "parts": parts,
            }	, )
        r.raise_for_status()

        write_msg({"event": "complete", "oid": oid} )











import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
  import torch

  from ..models.auto.modeling_auto import (
      MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
  )

if is_tf_available() and is_tensorflow_probability_available():
  import tensorflow as tf
  import tensorflow_probability as tfp

  from ..models.auto.modeling_tf_auto import (
      TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
  )
class      _UpperCamelCase (			lowerCAmelCase						):
      def __call__(     self				:Optional[int]	, lowerCamelCase				:Tuple=None	, lowerCamelCase				:Tuple=None	, **lowerCamelCase				:str )       ->     Optional[int]:
       # Returns tqa_pipeline_inputs of shape:
       # [
       #   {"table": pd.DataFrame, "query": List[str]},
       #   ...,
       #   {"table": pd.DataFrame, "query" : List[str]}
       # ]
       requires_backends(self	, "pandas" )
       import pandas as pd

       if table is None:
        raise ValueError("Keyword argument `table` cannot be None." )
       elif query is None:
        if isinstance(lowerCamelCase	, lowerCamelCase ) and table.get("query" ) is not None and table.get("table" ) is not None:
         UpperCAmelCase__								=  [table]
        elif isinstance(lowerCamelCase	, lowerCamelCase ) and len(lowerCamelCase ) > 0:
         if not all(isinstance(lowerCamelCase	, lowerCamelCase ) for d in table ):
          raise ValueError(
              f'''Keyword argument `table` should be a list of dict, but is {(type(lowerCamelCase ) for d in table)}''' )

         if table[0].get("query" ) is not None and table[0].get("table" ) is not None:
          UpperCAmelCase__								=  table
         else:
          raise ValueError(
              "If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"
              f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.''' )
        elif Dataset is not None and isinstance(lowerCamelCase	, lowerCamelCase ) or isinstance(lowerCamelCase	, types.GeneratorType ):
         return table
        else:
         raise ValueError(
             "Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but "
             f'''is {type(lowerCamelCase )})''' )
       else:
        UpperCAmelCase__								=  [{"table": table, "query": query}]

       for tqa_pipeline_input in tqa_pipeline_inputs:
        if not isinstance(tqa_pipeline_input["table"]	, pd.DataFrame ):
         if tqa_pipeline_input["table"] is None:
          raise ValueError("Table cannot be None." )

         UpperCAmelCase__								=  pd.DataFrame(tqa_pipeline_input["table"] )

       return tqa_pipeline_inputs



@add_end_docstrings(lowerCAmelCase						)
class      _UpperCamelCase (			lowerCAmelCase						):
      UpperCAmelCase_            =						"""table,query"""
      def __init__(     self				:int	, lowerCamelCase				:Any=TableQuestionAnsweringArgumentHandler()	, *lowerCamelCase				:Optional[int]	, **lowerCamelCase				:str )       ->     int:
       super().__init__(*lowerCamelCase	, **lowerCamelCase )
       UpperCAmelCase__								=  args_parser

       self.check_model_type(
           dict(
               TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
               + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items() )
           if self.framework == "tf"
           else dict(
               MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items() ) )

       UpperCAmelCase__								=  bool(getattr(self.model.config	, "aggregation_labels"	, lowerCamelCase ) ) and bool(
           getattr(self.model.config	, "num_aggregation_labels"	, lowerCamelCase ) )
       UpperCAmelCase__								=  "tapas" if hasattr(self.model.config	, "aggregation_labels" ) else None
      def 		UpperCAmelCase_  (     self				:int	, **lowerCamelCase				:Dict )       ->     Optional[int]:
       return self.model(**lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Dict	, **lowerCamelCase				:Optional[int] )       ->     Dict:
       if self.framework == "pt":
        UpperCAmelCase__								=  []
        UpperCAmelCase__								=  []
        UpperCAmelCase__								=  None
        UpperCAmelCase__								=  inputs["input_ids"].shape[0]

        UpperCAmelCase__								=  inputs["input_ids"].to(self.device )
        UpperCAmelCase__								=  inputs["attention_mask"].to(self.device )
        UpperCAmelCase__								=  inputs["token_type_ids"].to(self.device )
        UpperCAmelCase__								=  None

        for index in range(lowerCamelCase ):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
          UpperCAmelCase__								=  token_type_ids_example[:, 3]  # shape (seq_len,)
          UpperCAmelCase__								=  np.zeros_like(prev_labels_example.cpu().numpy() )  # shape (seq_len,)

          UpperCAmelCase__								=  token_type_ids[index]  # shape (seq_len, 7)
          for i in range(model_labels.shape[0] ):
           UpperCAmelCase__								=  token_type_ids_example[:, 0].tolist()[i]
           UpperCAmelCase__								=  token_type_ids_example[:, 1].tolist()[i] - 1
           UpperCAmelCase__								=  token_type_ids_example[:, 2].tolist()[i] - 1

           if row_id >= 0 and col_id >= 0 and segment_id == 1:
            UpperCAmelCase__								=  int(prev_answers[(col_id, row_id)] )

          UpperCAmelCase__								=  torch.from_numpy(lowerCamelCase ).type(torch.long ).to(self.device )

         UpperCAmelCase__								=  input_ids[index]
         UpperCAmelCase__								=  attention_mask[index]  # shape (seq_len,)
         UpperCAmelCase__								=  token_type_ids[index]  # shape (seq_len, 7)
         UpperCAmelCase__								=  self.model(
             input_ids=input_ids_example.unsqueeze(0 )	, attention_mask=attention_mask_example.unsqueeze(0 )	, token_type_ids=token_type_ids_example.unsqueeze(0 )	, )
         UpperCAmelCase__								=  outputs.logits

         if self.aggregate:
          all_aggregations.append(outputs.logits_aggregation )

         all_logits.append(lowerCamelCase )

         UpperCAmelCase__								=  torch.distributions.Bernoulli(logits=lowerCamelCase )
         UpperCAmelCase__								=  dist_per_token.probs * attention_mask_example.type(torch.floataa ).to(
             dist_per_token.probs.device )

         UpperCAmelCase__								=  collections.defaultdict(lowerCamelCase )
         for i, p in enumerate(probabilities.squeeze().tolist() ):
          UpperCAmelCase__								=  token_type_ids_example[:, 0].tolist()[i]
          UpperCAmelCase__								=  token_type_ids_example[:, 1].tolist()[i] - 1
          UpperCAmelCase__								=  token_type_ids_example[:, 2].tolist()[i] - 1
          if col >= 0 and row >= 0 and segment_id == 1:
           coords_to_probs[(col, row)].append(lowerCamelCase )

         UpperCAmelCase__								=  {key: np.array(coords_to_probs[key] ).mean() > 0.5 for key in coords_to_probs}

        UpperCAmelCase__								=  torch.cat(tuple(lowerCamelCase )	, 0 )

        return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(lowerCamelCase )	, 0 ))
       else:
        UpperCAmelCase__								=  []
        UpperCAmelCase__								=  []
        UpperCAmelCase__								=  None
        UpperCAmelCase__								=  inputs["input_ids"].shape[0]

        UpperCAmelCase__								=  inputs["input_ids"]
        UpperCAmelCase__								=  inputs["attention_mask"]
        UpperCAmelCase__								=  inputs["token_type_ids"].numpy()
        UpperCAmelCase__								=  None

        for index in range(lowerCamelCase ):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
          UpperCAmelCase__								=  token_type_ids_example[:, 3]  # shape (seq_len,)
          UpperCAmelCase__								=  np.zeros_like(lowerCamelCase	, dtype=np.intaa )  # shape (seq_len,)

          UpperCAmelCase__								=  token_type_ids[index]  # shape (seq_len, 7)
          for i in range(model_labels.shape[0] ):
           UpperCAmelCase__								=  token_type_ids_example[:, 0].tolist()[i]
           UpperCAmelCase__								=  token_type_ids_example[:, 1].tolist()[i] - 1
           UpperCAmelCase__								=  token_type_ids_example[:, 2].tolist()[i] - 1

           if row_id >= 0 and col_id >= 0 and segment_id == 1:
            UpperCAmelCase__								=  int(prev_answers[(col_id, row_id)] )

          UpperCAmelCase__								=  model_labels

         UpperCAmelCase__								=  input_ids[index]
         UpperCAmelCase__								=  attention_mask[index]  # shape (seq_len,)
         UpperCAmelCase__								=  token_type_ids[index]  # shape (seq_len, 7)
         UpperCAmelCase__								=  self.model(
             input_ids=np.expand_dims(lowerCamelCase	, axis=0 )	, attention_mask=np.expand_dims(lowerCamelCase	, axis=0 )	, token_type_ids=np.expand_dims(lowerCamelCase	, axis=0 )	, )
         UpperCAmelCase__								=  outputs.logits

         if self.aggregate:
          all_aggregations.append(outputs.logits_aggregation )

         all_logits.append(lowerCamelCase )

         UpperCAmelCase__								=  tfp.distributions.Bernoulli(logits=lowerCamelCase )
         UpperCAmelCase__								=  dist_per_token.probs_parameter() * tf.cast(lowerCamelCase	, tf.floataa )

         UpperCAmelCase__								=  collections.defaultdict(lowerCamelCase )
         UpperCAmelCase__								=  token_type_ids_example
         for i, p in enumerate(tf.squeeze(lowerCamelCase ).numpy().tolist() ):
          UpperCAmelCase__								=  token_type_ids_example[:, 0].tolist()[i]
          UpperCAmelCase__								=  token_type_ids_example[:, 1].tolist()[i] - 1
          UpperCAmelCase__								=  token_type_ids_example[:, 2].tolist()[i] - 1
          if col >= 0 and row >= 0 and segment_id == 1:
           coords_to_probs[(col, row)].append(lowerCamelCase )

         UpperCAmelCase__								=  {key: np.array(coords_to_probs[key] ).mean() > 0.5 for key in coords_to_probs}

        UpperCAmelCase__								=  tf.concat(tuple(lowerCamelCase )	, 0 )

        return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(lowerCamelCase )	, 0 ))
      def __call__(     self				:str	, *lowerCamelCase				:List[Any]	, **lowerCamelCase				:List[Any] )       ->     Dict:
       UpperCAmelCase__								=  self._args_parser(*lowerCamelCase	, **lowerCamelCase )

       UpperCAmelCase__								=  super().__call__(lowerCamelCase	, **lowerCamelCase )
       if len(lowerCamelCase ) == 1:
        return results[0]
       return results
      def 		UpperCAmelCase_  (     self				:List[str]	, lowerCamelCase				:int=None	, lowerCamelCase				:Optional[int]=None	, lowerCamelCase				:int=None	, **lowerCamelCase				:Tuple )       ->     Tuple:
       UpperCAmelCase__								=  {}
       if padding is not None:
        UpperCAmelCase__								=  padding
       if truncation is not None:
        UpperCAmelCase__								=  truncation

       UpperCAmelCase__								=  {}
       if sequential is not None:
        UpperCAmelCase__								=  sequential
       return preprocess_params, forward_params, {}
      def 		UpperCAmelCase_  (     self				:Optional[int]	, lowerCamelCase				:List[Any]	, lowerCamelCase				:Tuple=None	, lowerCamelCase				:Union[str, Any]=True	, lowerCamelCase				:str=None )       ->     Union[str, Any]:
       if truncation is None:
        if self.type == "tapas":
         UpperCAmelCase__								=  "drop_rows_to_fit"
        else:
         UpperCAmelCase__								=  "do_not_truncate"

       UpperCAmelCase__	,	UpperCAmelCase__								=  pipeline_input["table"], pipeline_input["query"]
       if table.empty:
        raise ValueError("table is empty" )
       if query is None or query == "":
        raise ValueError("query is empty" )
       UpperCAmelCase__								=  self.tokenizer(lowerCamelCase	, lowerCamelCase	, return_tensors=self.framework	, truncation=lowerCamelCase	, padding=lowerCamelCase )
       UpperCAmelCase__								=  table
       return inputs
      def 		UpperCAmelCase_  (     self				:Union[str, Any]	, lowerCamelCase				:Optional[int]	, lowerCamelCase				:Optional[Any]=False )       ->     int:
       UpperCAmelCase__								=  model_inputs.pop("table" )

       if self.type == "tapas":
        if sequential:
         UpperCAmelCase__								=  self.sequential_inference(**lowerCamelCase )
        else:
         UpperCAmelCase__								=  self.batch_inference(**lowerCamelCase )
       else:
        UpperCAmelCase__								=  self.model.generate(**lowerCamelCase )
       UpperCAmelCase__								=  {"model_inputs": model_inputs, "table": table, "outputs": outputs}
       return model_outputs







      def 		UpperCAmelCase_  (     self				:Any	, lowerCamelCase				:Optional[Any] )       ->     Dict:
       UpperCAmelCase__								=  model_outputs["model_inputs"]
       UpperCAmelCase__								=  model_outputs["table"]
       UpperCAmelCase__								=  model_outputs["outputs"]
       if self.type == "tapas":
        if self.aggregate:
         UpperCAmelCase__	,	UpperCAmelCase__								=  outputs[:2]
         UpperCAmelCase__								=  self.tokenizer.convert_logits_to_predictions(lowerCamelCase	, lowerCamelCase	, lowerCamelCase )
         UpperCAmelCase__	,	UpperCAmelCase__								=  predictions
         UpperCAmelCase__								=  {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(lowerCamelCase )}

         UpperCAmelCase__								=  self.model.config.no_aggregation_label_index
         UpperCAmelCase__								=  {
             i: aggregators[i] + " > " for i, pred in enumerate(lowerCamelCase ) if pred != no_agg_label_index
         }
        else:
         UpperCAmelCase__								=  outputs[0]
         UpperCAmelCase__								=  self.tokenizer.convert_logits_to_predictions(lowerCamelCase	, lowerCamelCase )
         UpperCAmelCase__								=  predictions[0]
         UpperCAmelCase__								=  {}
         UpperCAmelCase__								=  {}
        UpperCAmelCase__								=  []
        for index, coordinates in enumerate(lowerCamelCase ):
         UpperCAmelCase__								=  [table.iat[coordinate] for coordinate in coordinates]
         UpperCAmelCase__								=  aggregators.get(lowerCamelCase	, "" )
         UpperCAmelCase__								=  aggregators_prefix.get(lowerCamelCase	, "" )
         UpperCAmelCase__								=  {
             "answer": aggregator_prefix + ", ".join(lowerCamelCase ),
             "coordinates": coordinates,
             "cells": [table.iat[coordinate] for coordinate in coordinates],
         }
         if aggregator:
          UpperCAmelCase__								=  aggregator

         answers.append(lowerCamelCase )
        if len(lowerCamelCase ) == 0:
         raise PipelineException("Empty answer" )
       else:
        UpperCAmelCase__								=  [{"answer": answer} for answer in self.tokenizer.batch_decode(lowerCamelCase	, skip_special_tokens=lowerCamelCase )]

       return answers if len(lowerCamelCase ) > 1 else answers[0]





