






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__a :    List[Any]								=				logging.get_logger(__name__)

__a :    Any								=				{
    "google/bigbird-pegasus-large-arxiv": (
        "https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-pubmed": (
        "https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-bigpatent": (
        "https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json"
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}


class  __lowercase (		lowercase_					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			"bigbird_pegasus"
     SCREAMING_SNAKE_CASE				  =			["past_key_values"]
     SCREAMING_SNAKE_CASE				  =			{
         "num_attention_heads": "encoder_attention_heads",
         "hidden_size": "d_model",
         "attention_probs_dropout_prob": "attention_dropout",
     }




     def __init__(			self    :     Optional[int]	,		UpperCamelCase_    :     Union[str, Any]=96_103	,		UpperCamelCase_    :     Tuple=4_096	,		UpperCamelCase_    :     Dict=16	,		UpperCamelCase_    :     List[str]=4_096	,		UpperCamelCase_    :     Any=16	,		UpperCamelCase_    :     Dict=16	,		UpperCamelCase_    :     Optional[int]=4_096	,		UpperCamelCase_    :     str=16	,		UpperCamelCase_    :     Any=0.0	,		UpperCamelCase_    :     Optional[Any]=0.0	,		UpperCamelCase_    :     Any=True	,		UpperCamelCase_    :     int=True	,		UpperCamelCase_    :     Any="gelu_new"	,		UpperCamelCase_    :     int=1_024	,		UpperCamelCase_    :     Optional[int]=0.1	,		UpperCamelCase_    :     Any=0.0	,		UpperCamelCase_    :     List[str]=0.0	,		UpperCamelCase_    :     List[str]=0.02	,		UpperCamelCase_    :     List[str]=2	,		UpperCamelCase_    :     Optional[Any]=0.0	,		UpperCamelCase_    :     Union[str, Any]=True	,		UpperCamelCase_    :     str=0	,		UpperCamelCase_    :     str=2	,		UpperCamelCase_    :     Any=1	,		UpperCamelCase_    :     List[Any]="block_sparse"	,		UpperCamelCase_    :     Dict=64	,		UpperCamelCase_    :     str=3	,		UpperCamelCase_    :     Union[str, Any]=False	,		**UpperCamelCase_    :     Union[str, Any]	,		):


         """simple docstring"""



         __A             =		vocab_size
         __A             =		max_position_embeddings
         __A             =		d_model
         __A             =		encoder_ffn_dim
         __A             =		encoder_layers
         __A             =		encoder_attention_heads
         __A             =		decoder_ffn_dim
         __A             =		decoder_layers
         __A             =		decoder_attention_heads
         __A             =		dropout
         __A             =		attention_dropout
         __A             =		activation_dropout
         __A             =		activation_function
         __A             =		init_std
         __A             =		encoder_layerdrop
         __A             =		decoder_layerdrop
         __A             =		classifier_dropout
         __A             =		use_cache
         __A             =		encoder_layers
         __A             =		scale_embedding  # scale factor will be sqrt(d_model) if True

         # extra config
         __A             =		attention_type
         __A             =		block_size
         __A             =		num_random_blocks
         __A             =		use_bias

         super().__init__(
             pad_token_id=UpperCamelCase_	,		bos_token_id=UpperCamelCase_	,		eos_token_id=UpperCamelCase_	,		is_encoder_decoder=UpperCamelCase_	,		decoder_start_token_id=UpperCamelCase_	,		**UpperCamelCase_	,		)


class  __lowercase (		lowercase_					):







     '''simple docstring'''




     @property
     def 						lowerCAmelCase_       (			self    :     int			):


         """simple docstring"""



         if self.task in ["default", "seq2seq-lm"]:
             __A             =		OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                 ]			)

             if self.use_past:
                 __A             =		{0: """batch"""}
                 __A             =		{0: """batch""", 1: """past_decoder_sequence + sequence"""}
             else:
                 __A             =		{0: """batch""", 1: """decoder_sequence"""}
                 __A             =		{0: """batch""", 1: """decoder_sequence"""}

             if self.use_past:
                 self.fill_with_past_key_values_(UpperCamelCase_	,		direction="""inputs"""			)
         elif self.task == "causal-lm":
             # TODO: figure this case out.
             __A             =		OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                 ]			)
             if self.use_past:
                 __A      ,		__A             =		self.num_layers
                 for i in range(UpperCamelCase_			):
                     __A             =		{0: """batch""", 2: """past_sequence + sequence"""}
                     __A             =		{0: """batch""", 2: """past_sequence + sequence"""}
         else:
             __A             =		OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
                     ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
                 ]			)

         return common_inputs




     @property
     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         if self.task in ["default", "seq2seq-lm"]:
             __A             =		super().outputs
         else:
             __A             =		super(UpperCamelCase_	,		self			).outputs
             if self.use_past:
                 __A      ,		__A             =		self.num_layers
                 for i in range(UpperCamelCase_			):
                     __A             =		{0: """batch""", 2: """past_sequence + sequence"""}
                     __A             =		{0: """batch""", 2: """past_sequence + sequence"""}
         return common_outputs




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     PreTrainedTokenizer	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     bool = False	,		UpperCamelCase_    :     Optional[TensorType] = None	,		):


         """simple docstring"""



         __A             =		self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_			)

         # Generate decoder inputs
         __A             =		seq_length if not self.use_past else 1
         __A             =		self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_			)
         __A             =		{F"decoder_{name}": tensor for name, tensor in decoder_inputs.items()}
         __A             =		dict(**UpperCamelCase_	,		**UpperCamelCase_			)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""			)
             else:
                 import torch
             __A      ,		__A             =		common_inputs["""input_ids"""].shape
             __A             =		common_inputs["""decoder_input_ids"""].shape[1]
             __A      ,		__A             =		self.num_attention_heads
             __A             =		(
                 batch,
                 num_encoder_attention_heads,
                 encoder_seq_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             __A             =		decoder_seq_length + 3
             __A             =		(
                 batch,
                 num_decoder_attention_heads,
                 decoder_past_length,
                 self._config.hidden_size // num_decoder_attention_heads,
             )

             __A             =		torch.cat(
                 [common_inputs["""decoder_attention_mask"""], torch.ones(UpperCamelCase_	,		UpperCamelCase_			)]	,		dim=1			)

             __A             =		[]
             # If the number of encoder and decoder layers are present in the model configuration, both are considered
             __A      ,		__A             =		self.num_layers
             __A             =		min(UpperCamelCase_	,		UpperCamelCase_			)
             __A             =		max(UpperCamelCase_	,		UpperCamelCase_			) - min_num_layers
             __A             =		"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

             for _ in range(UpperCamelCase_			):
                 common_inputs["past_key_values"].append(
                     (
                         torch.zeros(UpperCamelCase_			),
                         torch.zeros(UpperCamelCase_			),
                         torch.zeros(UpperCamelCase_			),
                         torch.zeros(UpperCamelCase_			),
                     )			)
             # TODO: test this.
             __A             =		encoder_shape if remaining_side_name == """encoder""" else decoder_shape
             for _ in range(UpperCamelCase_	,		UpperCamelCase_			):
                 common_inputs["past_key_values"].append((torch.zeros(UpperCamelCase_			), torch.zeros(UpperCamelCase_			))			)
         return common_inputs




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     PreTrainedTokenizer	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     bool = False	,		UpperCamelCase_    :     Optional[TensorType] = None	,		):


         """simple docstring"""



         __A             =		self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_			)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""			)
             else:
                 import torch
             __A      ,		__A             =		common_inputs["""input_ids"""].shape
             # Not using the same length for past_key_values
             __A             =		seqlen + 2
             __A      ,		__A             =		self.num_layers
             __A      ,		__A             =		self.num_attention_heads
             __A             =		(
                 batch,
                 num_encoder_attention_heads,
                 past_key_values_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )

             __A             =		common_inputs["""attention_mask"""].dtype
             __A             =		torch.cat(
                 [common_inputs["""attention_mask"""], torch.ones(UpperCamelCase_	,		UpperCamelCase_	,		dtype=UpperCamelCase_			)]	,		dim=1			)
             __A             =		[
                 (torch.zeros(UpperCamelCase_			), torch.zeros(UpperCamelCase_			)) for _ in range(UpperCamelCase_			)
             ]
         return common_inputs




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]	,		UpperCamelCase_    :     PreTrainedTokenizer	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     bool = False	,		UpperCamelCase_    :     Optional[TensorType] = None	,		):


         """simple docstring"""



         __A             =		compute_effective_axis_dimension(
             UpperCamelCase_	,		fixed_dimension=OnnxConfig.default_fixed_batch	,		num_token_to_add=0			)

         # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
         __A             =		tokenizer.num_special_tokens_to_add(UpperCamelCase_			)
         __A             =		compute_effective_axis_dimension(
             UpperCamelCase_	,		fixed_dimension=OnnxConfig.default_fixed_sequence	,		num_token_to_add=UpperCamelCase_			)

         # Generate dummy inputs according to compute batch and sequence
         __A             =		[""" """.join([tokenizer.unk_token]			) * seq_length] * batch_size
         __A             =		dict(tokenizer(UpperCamelCase_	,		return_tensors=UpperCamelCase_			)			)
         return common_inputs




     def 						lowerCAmelCase_       (			self    :     str	,		UpperCamelCase_    :     PreTrainedTokenizer	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     int = -1	,		UpperCamelCase_    :     bool = False	,		UpperCamelCase_    :     Optional[TensorType] = None	,		):


         """simple docstring"""



         if self.task in ["default", "seq2seq-lm"]:
             __A             =		self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                 UpperCamelCase_	,		batch_size=UpperCamelCase_	,		seq_length=UpperCamelCase_	,		is_pair=UpperCamelCase_	,		framework=UpperCamelCase_			)

         elif self.task == "causal-lm":
             __A             =		self._generate_dummy_inputs_for_causal_lm(
                 UpperCamelCase_	,		batch_size=UpperCamelCase_	,		seq_length=UpperCamelCase_	,		is_pair=UpperCamelCase_	,		framework=UpperCamelCase_			)
         else:
             __A             =		self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 UpperCamelCase_	,		batch_size=UpperCamelCase_	,		seq_length=UpperCamelCase_	,		is_pair=UpperCamelCase_	,		framework=UpperCamelCase_			)

         return common_inputs





     def 						lowerCAmelCase_       (			self    :     List[Any]	,		UpperCamelCase_    :     Optional[int]	,		UpperCamelCase_    :     str	,		UpperCamelCase_    :     Union[str, Any]	,		UpperCamelCase_    :     Union[str, Any]			):


         """simple docstring"""



         if self.task in ["default", "seq2seq-lm"]:
             __A             =		super()._flatten_past_key_values_(UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_			)
         else:
             __A             =		super(UpperCamelCase_	,		self			)._flatten_past_key_values_(
                 UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_	,		UpperCamelCase_			)












import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__a :    List[str]								=				get_logger(__name__)


if TYPE_CHECKING:
  from .builder import DatasetBuilder



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Dict     ,			__lowercase				:     Optional[Union[str, bool]] = None      )  -> List[str]:





    """simple docstring"""

    __A             =		importlib.import_module(__lowercase      )

    # TODO(QL): always update the module to add subsequent new authentication
    if hasattr(__lowercase     ,			"""_patched_for_streaming"""      ) and module._patched_for_streaming:
        return

    def wrap_auth(__lowercase				:     int      ):
        @wraps(__lowercase      )
        def wrapper(*__lowercase				:     Tuple     ,			**__lowercase				:     List[str]      ):
            return function(*__lowercase     ,			use_auth_token=__lowercase     ,			**__lowercase      )

        __A             =		"""wrap_auth"""
        return wrapper

    # open files in a streaming fashion
    patch_submodule(__lowercase     ,			"""open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.listdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.walk"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""glob.glob"""     ,			wrap_auth(__lowercase      )      ).start()
    # allow to navigate in remote zip files
    patch_submodule(__lowercase     ,			"""os.path.join"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.dirname"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.basename"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.relpath"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.split"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.splitext"""     ,			__lowercase      ).start()
    # allow checks on paths
    patch_submodule(__lowercase     ,			"""os.path.exists"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isfile"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.getsize"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pathlib.Path"""     ,			__lowercase      ).start()
    # file readers
    patch_submodule(__lowercase     ,			"""gzip.open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""numpy.load"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_csv"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_excel"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""scipy.io.loadmat"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""xml.etree.ElementTree.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""xml.dom.minidom.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    __A             =		True






def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     "DatasetBuilder"      )  -> Union[str, Any]:





    """simple docstring"""

    extend_module_for_streaming(builder.__module__     ,			use_auth_token=builder.use_auth_token      )
    # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
    if not builder.__module__.startswith("""datasets."""      ):  # check that it's not a packaged builder like csv
        for imports in get_imports(inspect.getfile(builder.__class__      )      ):
            if imports[0] == "internal":
                __A             =		imports[1]
                __A             =		""".""".join(builder.__module__.split("""."""      )[:-1] + [internal_import_name]      )
                extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
    from .builder import DatasetBuilder

    __A             =		[
        cls.__module__
        for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
        if issubclass(cls     ,			__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
    ]  # check it's not a standard builder from datasets.builder
    for module in parent_builder_modules:
        extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )




