





"""simple docstring"""





import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
    import torch

    from ..models.auto.modeling_auto import (
        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
    )

if is_tf_available() and is_tensorflow_probability_available():
    import tensorflow as tf
    import tensorflow_probability as tfp

    from ..models.auto.modeling_tf_auto import (
        TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
    )





class       snake_case		(       __UpperCAmelCase       ):

      def __call__(      self       :Union[str, Any]						,							_lowerCamelCase       :Tuple=None						,							_lowerCamelCase       :Any=None						,							**_lowerCamelCase       :Union[str, Any]						):
       # Returns tqa_pipeline_inputs of shape:
       # [
       #   {"table": pd.DataFrame, "query": List[str]},
       #   ...,
       #   {"table": pd.DataFrame, "query" : List[str]}
       # ]
       requires_backends(self						,							'''pandas'''						)
       import pandas as pd

       if table is None:
        raise ValueError('''Keyword argument `table` cannot be None.'''						)
       elif query is None:
        if isinstance(_lowerCamelCase						,							_lowerCamelCase						) and table.get('''query'''						) is not None and table.get('''table'''						) is not None:
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	[table]
        elif isinstance(_lowerCamelCase						,							_lowerCamelCase						) and len(_lowerCamelCase						) > 0:
         if not all(isinstance(_lowerCamelCase						,							_lowerCamelCase						) for d in table						):
          raise ValueError(
              f'''Keyword argument `table` should be a list of dict, but is {(type(_lowerCamelCase						) for d in table)}'''						)

         if table[0].get('''query'''						) is not None and table[0].get('''table'''						) is not None:
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	table
         else:
          raise ValueError(
              '''If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'''
              f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''						)
        elif Dataset is not None and isinstance(_lowerCamelCase						,							_lowerCamelCase						) or isinstance(_lowerCamelCase						,							types.GeneratorType						):
         return table
        else:
         raise ValueError(
             '''Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '''
             f'''is {type(_lowerCamelCase						)})'''						)
       else:
        __SCREAMING_SNAKE_CASE			:      Any 							=	[{'''table''': table, '''query''': query}]

       for tqa_pipeline_input in tqa_pipeline_inputs:
        if not isinstance(tqa_pipeline_input['''table''']						,							pd.DataFrame						):
         if tqa_pipeline_input["table"] is None:
          raise ValueError('''Table cannot be None.'''						)

         __SCREAMING_SNAKE_CASE			:      str 							=	pd.DataFrame(tqa_pipeline_input['''table''']						)

       return tqa_pipeline_inputs







@add_end_docstrings(__UpperCAmelCase       )
class       snake_case		(       __UpperCAmelCase       ):
      lowerCamelCase__      =			'''table,query'''

      def __init__(      self       :Dict						,							_lowerCamelCase       :Dict=TableQuestionAnsweringArgumentHandler()						,							*_lowerCamelCase       :Tuple						,							**_lowerCamelCase       :Dict						):
       super().__init__(*_lowerCamelCase						,							**_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      int 							=	args_parser

       self.check_model_type(
           dict(
               TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
               + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()						)
           if self.framework == '''tf'''
           else dict(
               MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()						)						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	bool(getattr(self.model.config						,							'''aggregation_labels'''						,							_lowerCamelCase						)						) and bool(
           getattr(self.model.config						,							'''num_aggregation_labels'''						,							_lowerCamelCase						)						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	'''tapas''' if hasattr(self.model.config						,							'''aggregation_labels'''						) else None

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						,							**_lowerCamelCase       :List[Any]						):
       return self.model(**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							**_lowerCamelCase       :Any						):
       if self.framework == "pt":
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[]
        __SCREAMING_SNAKE_CASE			:      Tuple 							=	[]
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	None
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	inputs['''input_ids'''].shape[0]

        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	inputs['''input_ids'''].to(self.device						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	inputs['''attention_mask'''].to(self.device						)
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	inputs['''token_type_ids'''].to(self.device						)
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	None

        for index in range(_lowerCamelCase						):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
          __SCREAMING_SNAKE_CASE			:      Any 							=	token_type_ids_example[:, 3]  # shape (seq_len,)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	np.zeros_like(prev_labels_example.cpu().numpy()						)  # shape (seq_len,)

          __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	token_type_ids[index]  # shape (seq_len, 7)
          for i in range(model_labels.shape[0]						):
           __SCREAMING_SNAKE_CASE			:      List[str] 							=	token_type_ids_example[:, 0].tolist()[i]
           __SCREAMING_SNAKE_CASE			:      Dict 							=	token_type_ids_example[:, 1].tolist()[i] - 1
           __SCREAMING_SNAKE_CASE			:      int 							=	token_type_ids_example[:, 2].tolist()[i] - 1

           if row_id >= 0 and col_id >= 0 and segment_id == 1:
            __SCREAMING_SNAKE_CASE			:      List[str] 							=	int(prev_answers[(col_id, row_id)]						)

          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	torch.from_numpy(_lowerCamelCase						).type(torch.long						).to(self.device						)

         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	input_ids[index]
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	attention_mask[index]  # shape (seq_len,)
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	token_type_ids[index]  # shape (seq_len, 7)
         __SCREAMING_SNAKE_CASE			:      Any 							=	self.model(
             input_ids=input_ids_example.unsqueeze(0						)						,							attention_mask=attention_mask_example.unsqueeze(0						)						,							token_type_ids=token_type_ids_example.unsqueeze(0						)						,							)
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	outputs.logits

         if self.aggregate:
          all_aggregations.append(outputs.logits_aggregation						)

         all_logits.append(_lowerCamelCase						)

         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	torch.distributions.Bernoulli(logits=_lowerCamelCase						)
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	dist_per_token.probs * attention_mask_example.type(torch.floataa						).to(
             dist_per_token.probs.device						)

         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	collections.defaultdict(_lowerCamelCase						)
         for i, p in enumerate(probabilities.squeeze().tolist()						):
          __SCREAMING_SNAKE_CASE			:      Any 							=	token_type_ids_example[:, 0].tolist()[i]
          __SCREAMING_SNAKE_CASE			:      Any 							=	token_type_ids_example[:, 1].tolist()[i] - 1
          __SCREAMING_SNAKE_CASE			:      str 							=	token_type_ids_example[:, 2].tolist()[i] - 1
          if col >= 0 and row >= 0 and segment_id == 1:
           coords_to_probs[(col, row)].append(_lowerCamelCase						)

         __SCREAMING_SNAKE_CASE			:      Tuple 							=	{key: np.array(coords_to_probs[key]						).mean() > 0.5 for key in coords_to_probs}

        __SCREAMING_SNAKE_CASE			:      Dict 							=	torch.cat(tuple(_lowerCamelCase						)						,							0						)

        return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(_lowerCamelCase						)						,							0						))
       else:
        __SCREAMING_SNAKE_CASE			:      int 							=	[]
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	[]
        __SCREAMING_SNAKE_CASE			:      int 							=	None
        __SCREAMING_SNAKE_CASE			:      Any 							=	inputs['''input_ids'''].shape[0]

        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	inputs['''input_ids''']
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	inputs['''attention_mask''']
        __SCREAMING_SNAKE_CASE			:      str 							=	inputs['''token_type_ids'''].numpy()
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	None

        for index in range(_lowerCamelCase						):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
          __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	token_type_ids_example[:, 3]  # shape (seq_len,)
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	np.zeros_like(_lowerCamelCase						,							dtype=np.intaa						)  # shape (seq_len,)

          __SCREAMING_SNAKE_CASE			:      int 							=	token_type_ids[index]  # shape (seq_len, 7)
          for i in range(model_labels.shape[0]						):
           __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	token_type_ids_example[:, 0].tolist()[i]
           __SCREAMING_SNAKE_CASE			:      List[str] 							=	token_type_ids_example[:, 1].tolist()[i] - 1
           __SCREAMING_SNAKE_CASE			:      str 							=	token_type_ids_example[:, 2].tolist()[i] - 1

           if row_id >= 0 and col_id >= 0 and segment_id == 1:
            __SCREAMING_SNAKE_CASE			:      str 							=	int(prev_answers[(col_id, row_id)]						)

          __SCREAMING_SNAKE_CASE			:      Dict 							=	model_labels

         __SCREAMING_SNAKE_CASE			:      Dict 							=	input_ids[index]
         __SCREAMING_SNAKE_CASE			:      Any 							=	attention_mask[index]  # shape (seq_len,)
         __SCREAMING_SNAKE_CASE			:      str 							=	token_type_ids[index]  # shape (seq_len, 7)
         __SCREAMING_SNAKE_CASE			:      Dict 							=	self.model(
             input_ids=np.expand_dims(_lowerCamelCase						,							axis=0						)						,							attention_mask=np.expand_dims(_lowerCamelCase						,							axis=0						)						,							token_type_ids=np.expand_dims(_lowerCamelCase						,							axis=0						)						,							)
         __SCREAMING_SNAKE_CASE			:      Dict 							=	outputs.logits

         if self.aggregate:
          all_aggregations.append(outputs.logits_aggregation						)

         all_logits.append(_lowerCamelCase						)

         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tfp.distributions.Bernoulli(logits=_lowerCamelCase						)
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	dist_per_token.probs_parameter() * tf.cast(_lowerCamelCase						,							tf.floataa						)

         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	collections.defaultdict(_lowerCamelCase						)
         __SCREAMING_SNAKE_CASE			:      Any 							=	token_type_ids_example
         for i, p in enumerate(tf.squeeze(_lowerCamelCase						).numpy().tolist()						):
          __SCREAMING_SNAKE_CASE			:      int 							=	token_type_ids_example[:, 0].tolist()[i]
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	token_type_ids_example[:, 1].tolist()[i] - 1
          __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	token_type_ids_example[:, 2].tolist()[i] - 1
          if col >= 0 and row >= 0 and segment_id == 1:
           coords_to_probs[(col, row)].append(_lowerCamelCase						)

         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	{key: np.array(coords_to_probs[key]						).mean() > 0.5 for key in coords_to_probs}

        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	tf.concat(tuple(_lowerCamelCase						)						,							0						)

        return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(_lowerCamelCase						)						,							0						))

      def __call__(      self       :List[str]						,							*_lowerCamelCase       :Dict						,							**_lowerCamelCase       :Any						):
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self._args_parser(*_lowerCamelCase						,							**_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      List[str] 							=	super().__call__(_lowerCamelCase						,							**_lowerCamelCase						)
       if len(_lowerCamelCase						) == 1:
        return results[0]
       return results

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :Dict=None						,							_lowerCamelCase       :Optional[int]=None						,							_lowerCamelCase       :Optional[Any]=None						,							**_lowerCamelCase       :str						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	{}
       if padding is not None:
        __SCREAMING_SNAKE_CASE			:      int 							=	padding
       if truncation is not None:
        __SCREAMING_SNAKE_CASE			:      int 							=	truncation

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	{}
       if sequential is not None:
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	sequential
       return preprocess_params, forward_params, {}

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :Tuple						,							_lowerCamelCase       :Tuple=None						,							_lowerCamelCase       :int=True						,							_lowerCamelCase       :Dict=None						):
       if truncation is None:
        if self.type == "tapas":
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	'''drop_rows_to_fit'''
        else:
         __SCREAMING_SNAKE_CASE			:      int 							=	'''do_not_truncate'''

       __SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE			:      Optional[int] 							=	pipeline_input['''table'''], pipeline_input['''query''']
       if table.empty:
        raise ValueError('''table is empty'''						)
       if query is None or query == "":
        raise ValueError('''query is empty'''						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.tokenizer(_lowerCamelCase						,							_lowerCamelCase						,							return_tensors=self.framework						,							truncation=_lowerCamelCase						,							padding=_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	table
       return inputs

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :str						,							_lowerCamelCase       :Dict=False						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	model_inputs.pop('''table'''						)

       if self.type == "tapas":
        if sequential:
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.sequential_inference(**_lowerCamelCase						)
        else:
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self.batch_inference(**_lowerCamelCase						)
       else:
        __SCREAMING_SNAKE_CASE			:      Any 							=	self.model.generate(**_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	{'''model_inputs''': model_inputs, '''table''': table, '''outputs''': outputs}
       return model_outputs



      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						,							_lowerCamelCase       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      str 							=	model_outputs['''model_inputs''']
       __SCREAMING_SNAKE_CASE			:      str 							=	model_outputs['''table''']
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	model_outputs['''outputs''']
       if self.type == "tapas":
        if self.aggregate:
         __SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE			:      Optional[int] 							=	outputs[:2]
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.tokenizer.convert_logits_to_predictions(_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						)
         __SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE			:      Any 							=	predictions
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(_lowerCamelCase						)}

         __SCREAMING_SNAKE_CASE			:      str 							=	self.model.config.no_aggregation_label_index
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	{
             i: aggregators[i] + ''' > ''' for i, pred in enumerate(_lowerCamelCase						) if pred != no_agg_label_index
         }
        else:
         __SCREAMING_SNAKE_CASE			:      Dict 							=	outputs[0]
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self.tokenizer.convert_logits_to_predictions(_lowerCamelCase						,							_lowerCamelCase						)
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	predictions[0]
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	{}
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	{}
        __SCREAMING_SNAKE_CASE			:      str 							=	[]
        for index, coordinates in enumerate(_lowerCamelCase						):
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[table.iat[coordinate] for coordinate in coordinates]
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	aggregators.get(_lowerCamelCase						,							''''''						)
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	aggregators_prefix.get(_lowerCamelCase						,							''''''						)
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	{
             '''answer''': aggregator_prefix + ''', '''.join(_lowerCamelCase						),
             '''coordinates''': coordinates,
             '''cells''': [table.iat[coordinate] for coordinate in coordinates],
         }
         if aggregator:
          __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	aggregator

         answers.append(_lowerCamelCase						)
        if len(_lowerCamelCase						) == 0:
         raise PipelineException('''Empty answer'''						)
       else:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[{'''answer''': answer} for answer in self.tokenizer.batch_decode(_lowerCamelCase						,							skip_special_tokens=_lowerCamelCase						)]

       return answers if len(_lowerCamelCase						) > 1 else answers[0]








"""simple docstring"""





from ..utils import DummyObject, requires_backends





class       snake_case		(       metaclass=__UpperCAmelCase       ):
      lowerCamelCase__      =			['''torch''', '''transformers''', '''invisible_watermark''']

      def __init__(      self       :Optional[int]						,							*_lowerCamelCase       :int						,							**_lowerCamelCase       :Any						):
       requires_backends(self						,							['''torch''', '''transformers''', '''invisible_watermark''']						)

      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :Dict						,							*_lowerCamelCase       :Optional[Any]						,							**_lowerCamelCase       :int						):
       requires_backends(cls						,							['''torch''', '''transformers''', '''invisible_watermark''']						)



      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :Optional[int]						,							*_lowerCamelCase       :Optional[Any]						,							**_lowerCamelCase       :Union[str, Any]						):
       requires_backends(cls						,							['''torch''', '''transformers''', '''invisible_watermark''']						)







class       snake_case		(       metaclass=__UpperCAmelCase       ):
      lowerCamelCase__      =			['''torch''', '''transformers''', '''invisible_watermark''']

      def __init__(      self       :Dict						,							*_lowerCamelCase       :Dict						,							**_lowerCamelCase       :Optional[Any]						):
       requires_backends(self						,							['''torch''', '''transformers''', '''invisible_watermark''']						)

      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :Optional[int]						,							*_lowerCamelCase       :int						,							**_lowerCamelCase       :Optional[int]						):
       requires_backends(cls						,							['''torch''', '''transformers''', '''invisible_watermark''']						)



      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :List[Any]						,							*_lowerCamelCase       :Dict						,							**_lowerCamelCase       :List[str]						):
       requires_backends(cls						,							['''torch''', '''transformers''', '''invisible_watermark''']						)

