




'''simple docstring'''







import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
   import torch

   from ..models.auto.modeling_auto import (
       MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
       MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
   )

if is_tf_available() and is_tensorflow_probability_available():
   import tensorflow as tf
   import tensorflow_probability as tfp

   from ..models.auto.modeling_tf_auto import (
       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
       TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
   )


class       _a  (     __a			):
       def __call__(     self						:     Dict       ,    lowercase						:     List[str]=None       ,    lowercase						:     Any=None       ,    **lowercase						:     List[str]    ):



           '''simple docstring'''

           requires_backends(self       ,    '''pandas'''    )
           import pandas as pd

           if table is None:
               raise ValueError('''Keyword argument `table` cannot be None.'''    )
           elif query is None:
               if isinstance(lowercase       ,    lowercase    ) and table.get('''query'''    ) is not None and table.get('''table'''    ) is not None:
                   UpperCAmelCase      	=		[table]
               elif isinstance(lowercase       ,    lowercase    ) and len(lowercase    ) > 0:
                   if not all(isinstance(lowercase       ,    lowercase    ) for d in table    ):
                       raise ValueError(
                           f"Keyword argument `table` should be a list of dict, but is {(type(lowercase    ) for d in table)}"    )

                   if table[0].get('''query'''    ) is not None and table[0].get('''table'''    ) is not None:
                       UpperCAmelCase      	=		table
                   else:
                       raise ValueError(
                           '''If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'''
                           f" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys."    )
               elif Dataset is not None and isinstance(lowercase       ,    lowercase    ) or isinstance(lowercase       ,    types.GeneratorType    ):
                   return table
               else:
                   raise ValueError(
                       '''Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '''
                       f"is {type(lowercase    )})"    )
           else:
               UpperCAmelCase      	=		[{'''table''': table, '''query''': query}]

           for tqa_pipeline_input in tqa_pipeline_inputs:
               if not isinstance(tqa_pipeline_input['''table''']       ,    pd.DataFrame    ):
                   if tqa_pipeline_input["table"] is None:
                       raise ValueError('''Table cannot be None.'''    )

                   UpperCAmelCase      	=		pd.DataFrame(tqa_pipeline_input['''table''']    )

           return tqa_pipeline_inputs






@add_end_docstrings(__a			)
class       _a  (     __a			):
       __a    :					int           = """table,query"""
       def __init__(     self						:     Optional[Any]       ,    lowercase						:     int=TableQuestionAnsweringArgumentHandler()       ,    *lowercase						:     int       ,    **lowercase						:     Dict    ):



           '''simple docstring'''

           super().__init__(*lowercase       ,    **lowercase    )
           UpperCAmelCase      	=		args_parser

           self.check_model_type(
               dict(
                   TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
                   + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()    )
               if self.framework == '''tf'''
               else dict(
                   MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()    )    )

           UpperCAmelCase      	=		bool(getattr(self.model.config       ,    '''aggregation_labels'''       ,    lowercase    )    ) and bool(
               getattr(self.model.config       ,    '''num_aggregation_labels'''       ,    lowercase    )    )
           UpperCAmelCase      	=		'''tapas''' if hasattr(self.model.config       ,    '''aggregation_labels'''    ) else None
       def    A				(     self						:     Union[str, Any]       ,    **lowercase						:     List[str]    ):



           '''simple docstring'''

           return self.model(**lowercase    )
       def    A				(     self						:     Any       ,    **lowercase						:     Dict    ):



           '''simple docstring'''

           if self.framework == "pt":
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		inputs['''input_ids'''].shape[0]

               UpperCAmelCase      	=		inputs['''input_ids'''].to(self.device    )
               UpperCAmelCase      	=		inputs['''attention_mask'''].to(self.device    )
               UpperCAmelCase      	=		inputs['''token_type_ids'''].to(self.device    )
               UpperCAmelCase      	=		None

               for index in range(lowercase    ):
                   # If sequences have already been processed, the token type IDs will be created according to the previous
                   # answer.
                   if prev_answers is not None:
                       UpperCAmelCase      	=		token_type_ids_example[:, 3]  # shape (seq_len,)
                       UpperCAmelCase      	=		np.zeros_like(prev_labels_example.cpu().numpy()    )  # shape (seq_len,)

                       UpperCAmelCase      	=		token_type_ids[index]  # shape (seq_len, 7)
                       for i in range(model_labels.shape[0]    ):
                           UpperCAmelCase      	=		token_type_ids_example[:, 0].tolist()[i]
                           UpperCAmelCase      	=		token_type_ids_example[:, 1].tolist()[i] - 1
                           UpperCAmelCase      	=		token_type_ids_example[:, 2].tolist()[i] - 1

                           if row_id >= 0 and col_id >= 0 and segment_id == 1:
                               UpperCAmelCase      	=		int(prev_answers[(col_id, row_id)]    )

                       UpperCAmelCase      	=		torch.from_numpy(lowercase    ).type(torch.long    ).to(self.device    )

                   UpperCAmelCase      	=		input_ids[index]
                   UpperCAmelCase      	=		attention_mask[index]  # shape (seq_len,)
                   UpperCAmelCase      	=		token_type_ids[index]  # shape (seq_len, 7)
                   UpperCAmelCase      	=		self.model(
                       input_ids=input_ids_example.unsqueeze(0    )       ,    attention_mask=attention_mask_example.unsqueeze(0    )       ,    token_type_ids=token_type_ids_example.unsqueeze(0    )       ,    )
                   UpperCAmelCase      	=		outputs.logits

                   if self.aggregate:
                       all_aggregations.append(outputs.logits_aggregation    )

                   all_logits.append(lowercase    )

                   UpperCAmelCase      	=		torch.distributions.Bernoulli(logits=lowercase    )
                   UpperCAmelCase      	=		dist_per_token.probs * attention_mask_example.type(torch.floataa    ).to(
                       dist_per_token.probs.device    )

                   UpperCAmelCase      	=		collections.defaultdict(lowercase    )
                   for i, p in enumerate(probabilities.squeeze().tolist()    ):
                       UpperCAmelCase      	=		token_type_ids_example[:, 0].tolist()[i]
                       UpperCAmelCase      	=		token_type_ids_example[:, 1].tolist()[i] - 1
                       UpperCAmelCase      	=		token_type_ids_example[:, 2].tolist()[i] - 1
                       if col >= 0 and row >= 0 and segment_id == 1:
                           coords_to_probs[(col, row)].append(lowercase    )

                   UpperCAmelCase      	=		{key: np.array(coords_to_probs[key]    ).mean() > 0.5 for key in coords_to_probs}

               UpperCAmelCase      	=		torch.cat(tuple(lowercase    )       ,    0    )

               return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(lowercase    )       ,    0    ))
           else:
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		inputs['''input_ids'''].shape[0]

               UpperCAmelCase      	=		inputs['''input_ids''']
               UpperCAmelCase      	=		inputs['''attention_mask''']
               UpperCAmelCase      	=		inputs['''token_type_ids'''].numpy()
               UpperCAmelCase      	=		None

               for index in range(lowercase    ):
                   # If sequences have already been processed, the token type IDs will be created according to the previous
                   # answer.
                   if prev_answers is not None:
                       UpperCAmelCase      	=		token_type_ids_example[:, 3]  # shape (seq_len,)
                       UpperCAmelCase      	=		np.zeros_like(lowercase       ,    dtype=np.intaa    )  # shape (seq_len,)

                       UpperCAmelCase      	=		token_type_ids[index]  # shape (seq_len, 7)
                       for i in range(model_labels.shape[0]    ):
                           UpperCAmelCase      	=		token_type_ids_example[:, 0].tolist()[i]
                           UpperCAmelCase      	=		token_type_ids_example[:, 1].tolist()[i] - 1
                           UpperCAmelCase      	=		token_type_ids_example[:, 2].tolist()[i] - 1

                           if row_id >= 0 and col_id >= 0 and segment_id == 1:
                               UpperCAmelCase      	=		int(prev_answers[(col_id, row_id)]    )

                       UpperCAmelCase      	=		model_labels

                   UpperCAmelCase      	=		input_ids[index]
                   UpperCAmelCase      	=		attention_mask[index]  # shape (seq_len,)
                   UpperCAmelCase      	=		token_type_ids[index]  # shape (seq_len, 7)
                   UpperCAmelCase      	=		self.model(
                       input_ids=np.expand_dims(lowercase       ,    axis=0    )       ,    attention_mask=np.expand_dims(lowercase       ,    axis=0    )       ,    token_type_ids=np.expand_dims(lowercase       ,    axis=0    )       ,    )
                   UpperCAmelCase      	=		outputs.logits

                   if self.aggregate:
                       all_aggregations.append(outputs.logits_aggregation    )

                   all_logits.append(lowercase    )

                   UpperCAmelCase      	=		tfp.distributions.Bernoulli(logits=lowercase    )
                   UpperCAmelCase      	=		dist_per_token.probs_parameter() * tf.cast(lowercase       ,    tf.floataa    )

                   UpperCAmelCase      	=		collections.defaultdict(lowercase    )
                   UpperCAmelCase      	=		token_type_ids_example
                   for i, p in enumerate(tf.squeeze(lowercase    ).numpy().tolist()    ):
                       UpperCAmelCase      	=		token_type_ids_example[:, 0].tolist()[i]
                       UpperCAmelCase      	=		token_type_ids_example[:, 1].tolist()[i] - 1
                       UpperCAmelCase      	=		token_type_ids_example[:, 2].tolist()[i] - 1
                       if col >= 0 and row >= 0 and segment_id == 1:
                           coords_to_probs[(col, row)].append(lowercase    )

                   UpperCAmelCase      	=		{key: np.array(coords_to_probs[key]    ).mean() > 0.5 for key in coords_to_probs}

               UpperCAmelCase      	=		tf.concat(tuple(lowercase    )       ,    0    )

               return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(lowercase    )       ,    0    ))
       def __call__(     self						:     Tuple       ,    *lowercase						:     Union[str, Any]       ,    **lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self._args_parser(*lowercase       ,    **lowercase    )

           UpperCAmelCase      	=		super().__call__(lowercase       ,    **lowercase    )
           if len(lowercase    ) == 1:
               return results[0]
           return results
       def    A				(     self						:     str       ,    lowercase						:     Optional[int]=None       ,    lowercase						:     Union[str, Any]=None       ,    lowercase						:     Any=None       ,    **lowercase						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		{}
           if padding is not None:
               UpperCAmelCase      	=		padding
           if truncation is not None:
               UpperCAmelCase      	=		truncation

           UpperCAmelCase      	=		{}
           if sequential is not None:
               UpperCAmelCase      	=		sequential
           return preprocess_params, forward_params, {}
       def    A				(     self						:     Tuple       ,    lowercase						:     List[str]       ,    lowercase						:     Optional[Any]=None       ,    lowercase						:     Optional[Any]=True       ,    lowercase						:     List[Any]=None    ):



           '''simple docstring'''

           if truncation is None:
               if self.type == "tapas":
                   UpperCAmelCase      	=		'''drop_rows_to_fit'''
               else:
                   UpperCAmelCase      	=		'''do_not_truncate'''

           UpperCAmelCase	,					UpperCAmelCase      	=		pipeline_input['''table'''], pipeline_input['''query''']
           if table.empty:
               raise ValueError('''table is empty'''    )
           if query is None or query == "":
               raise ValueError('''query is empty'''    )
           UpperCAmelCase      	=		self.tokenizer(lowercase       ,    lowercase       ,    return_tensors=self.framework       ,    truncation=lowercase       ,    padding=lowercase    )
           UpperCAmelCase      	=		table
           return inputs
       def    A				(     self						:     str       ,    lowercase						:     Tuple       ,    lowercase						:     Tuple=False    ):



           '''simple docstring'''

           UpperCAmelCase      	=		model_inputs.pop('''table'''    )

           if self.type == "tapas":
               if sequential:
                   UpperCAmelCase      	=		self.sequential_inference(**lowercase    )
               else:
                   UpperCAmelCase      	=		self.batch_inference(**lowercase    )
           else:
               UpperCAmelCase      	=		self.model.generate(**lowercase    )
           UpperCAmelCase      	=		{'''model_inputs''': model_inputs, '''table''': table, '''outputs''': outputs}
           return model_outputs







       def    A				(     self						:     Optional[int]       ,    lowercase						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		model_outputs['''model_inputs''']
           UpperCAmelCase      	=		model_outputs['''table''']
           UpperCAmelCase      	=		model_outputs['''outputs''']
           if self.type == "tapas":
               if self.aggregate:
                   UpperCAmelCase	,					UpperCAmelCase      	=		outputs[:2]
                   UpperCAmelCase      	=		self.tokenizer.convert_logits_to_predictions(lowercase       ,    lowercase       ,    lowercase    )
                   UpperCAmelCase	,					UpperCAmelCase      	=		predictions
                   UpperCAmelCase      	=		{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(lowercase    )}

                   UpperCAmelCase      	=		self.model.config.no_aggregation_label_index
                   UpperCAmelCase      	=		{
                       i: aggregators[i] + ''' > ''' for i, pred in enumerate(lowercase    ) if pred != no_agg_label_index
                   }
               else:
                   UpperCAmelCase      	=		outputs[0]
                   UpperCAmelCase      	=		self.tokenizer.convert_logits_to_predictions(lowercase       ,    lowercase    )
                   UpperCAmelCase      	=		predictions[0]
                   UpperCAmelCase      	=		{}
                   UpperCAmelCase      	=		{}
               UpperCAmelCase      	=		[]
               for index, coordinates in enumerate(lowercase    ):
                   UpperCAmelCase      	=		[table.iat[coordinate] for coordinate in coordinates]
                   UpperCAmelCase      	=		aggregators.get(lowercase       ,    ''''''    )
                   UpperCAmelCase      	=		aggregators_prefix.get(lowercase       ,    ''''''    )
                   UpperCAmelCase      	=		{
                       '''answer''': aggregator_prefix + ''', '''.join(lowercase    ),
                       '''coordinates''': coordinates,
                       '''cells''': [table.iat[coordinate] for coordinate in coordinates],
                   }
                   if aggregator:
                       UpperCAmelCase      	=		aggregator

                   answers.append(lowercase    )
               if len(lowercase    ) == 0:
                   raise PipelineException('''Empty answer'''    )
           else:
               UpperCAmelCase      	=		[{'''answer''': answer} for answer in self.tokenizer.batch_decode(lowercase       ,    skip_special_tokens=lowercase    )]

           return answers if len(lowercase    ) > 1 else answers[0]










'''simple docstring'''







import argparse
import collections
from pathlib import Path

import torch
from torch.serialization import default_restore_location

from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader


A			   =collections.namedtuple(
    'CheckpointState', ['model_dict', 'optimizer_dict', 'scheduler_dict', 'offset', 'epoch', 'encoder_params']
)





def 						snake_case_							(_a     :							str					):
    print(F"Reading saved model from {model_file}"					)
    UpperCAmelCase      	=		torch.load(_a   ,       map_location=lambda _a   ,       _a					: default_restore_location(_a   ,       '''cpu'''					)					)
    return CheckpointState(**_a					)


class       _a  :
       def __init__(     self						:     List[Any]       ,    lowercase						:     Path    ):



           '''simple docstring'''

           UpperCAmelCase      	=		src_file
       def    A				(     self						:     str    ):



           '''simple docstring'''

           raise NotImplementedError







       @staticmethod
       def    A				(     lowercase						:     str       ,    *lowercase						:     Tuple       ,    **lowercase						:     Union[str, Any]    ):



           '''simple docstring'''

           if comp_type.startswith('''c'''    ):
               return DPRContextEncoderState(*lowercase       ,    **lowercase    )
           if comp_type.startswith('''q'''    ):
               return DPRQuestionEncoderState(*lowercase       ,    **lowercase    )
           if comp_type.startswith('''r'''    ):
               return DPRReaderState(*lowercase       ,    **lowercase    )
           else:
               raise ValueError('''Component type must be either \'ctx_encoder\', \'question_encoder\' or \'reader\'.'''    )






class       _a  (     __a			):
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		DPRContextEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''    )[0]    )    )
           print(f"Loading DPR biencoder from {self.src_file}"    )
           UpperCAmelCase      	=		load_states_from_checkpoint(self.src_file    )
           UpperCAmelCase	,					UpperCAmelCase      	=		model.ctx_encoder, '''ctx_model.'''
           # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
           UpperCAmelCase      	=		{'''bert_model.embeddings.position_ids''': model.ctx_encoder.bert_model.embeddings.position_ids}
           for key, value in saved_state.model_dict.items():
               if key.startswith(lowercase    ):
                   UpperCAmelCase      	=		key[len(lowercase    ) :]
                   if not key.startswith('''encode_proj.'''    ):
                       UpperCAmelCase      	=		'''bert_model.''' + key
                   UpperCAmelCase      	=		value
           encoder.load_state_dict(lowercase    )
           return model






class       _a  (     __a			):
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		DPRQuestionEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''    )[0]    )    )
           print(f"Loading DPR biencoder from {self.src_file}"    )
           UpperCAmelCase      	=		load_states_from_checkpoint(self.src_file    )
           UpperCAmelCase	,					UpperCAmelCase      	=		model.question_encoder, '''question_model.'''
           # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
           UpperCAmelCase      	=		{'''bert_model.embeddings.position_ids''': model.question_encoder.bert_model.embeddings.position_ids}
           for key, value in saved_state.model_dict.items():
               if key.startswith(lowercase    ):
                   UpperCAmelCase      	=		key[len(lowercase    ) :]
                   if not key.startswith('''encode_proj.'''    ):
                       UpperCAmelCase      	=		'''bert_model.''' + key
                   UpperCAmelCase      	=		value
           encoder.load_state_dict(lowercase    )
           return model






class       _a  (     __a			):
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		DPRReader(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''    )[0]    )    )
           print(f"Loading DPR reader from {self.src_file}"    )
           UpperCAmelCase      	=		load_states_from_checkpoint(self.src_file    )
           # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
           UpperCAmelCase      	=		{
               '''encoder.bert_model.embeddings.position_ids''': model.span_predictor.encoder.bert_model.embeddings.position_ids
           }
           for key, value in saved_state.model_dict.items():
               if key.startswith('''encoder.'''    ) and not key.startswith('''encoder.encode_proj'''    ):
                   UpperCAmelCase      	=		'''encoder.bert_model.''' + key[len('''encoder.'''    ) :]
               UpperCAmelCase      	=		value
           model.span_predictor.load_state_dict(lowercase    )
           return model






def 						snake_case_							(_a     :							str   ,       _a     :							Path   ,       _a     :							Path					):
    UpperCAmelCase      	=		Path(_a					)
    dest_dir.mkdir(exist_ok=_a					)

    UpperCAmelCase      	=		DPRState.from_type(_a   ,       src_file=_a					)
    UpperCAmelCase      	=		dpr_state.load_dpr_model()
    model.save_pretrained(_a					)
    model.from_pretrained(_a					)  # sanity check


if __name__ == "__main__":
   A			   =argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--type', type=str, help='Type of the component to convert: \'ctx_encoder\', \'question_encoder\' or \'reader\'.'
   )
   parser.add_argument(
       '--src',
       type=str,
       help=(
           'Path to the dpr checkpoint file. They can be downloaded from the official DPR repo'
           ' https://github.com/facebookresearch/DPR. Note that in the official repo, both encoders are stored in the'
           ' \'retriever\' checkpoints.'
       ),
   )
   parser.add_argument('--dest', type=str, default=None, help='Path to the output PyTorch model directory.')
   A			   =parser.parse_args()

   A			   =Path(args.src)
   A			   =f"""converted-{src_file.name}""" if args.dest is None else args.dest
   A			   =Path(dest_dir)
   assert src_file.exists()
   assert (
       args.type is not None
   ), "Please specify the component type of the DPR model to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
   convert(args.type, src_file, dest_dir)




