
"""simple docstring"""


import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase								=						datasets.utils.logging.get_logger(__name__)

@dataclass
class    A_     (					datasets.BuilderConfig       ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_     				=       None
							SCREAMING_SNAKE_CASE_     				=       "utf-8"
							SCREAMING_SNAKE_CASE_     				=       None
							SCREAMING_SNAKE_CASE_     				=       10 << 20  # 10MB
							SCREAMING_SNAKE_CASE_     				=       False
							SCREAMING_SNAKE_CASE_     				=       "line"






class    A_     (					datasets.ArrowBasedBuilder       ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_     				=       TextConfig





							def 				UpperCAmelCase__							(  self  :Optional[Any]	):





										"""simple docstring"""

										return datasets.DatasetInfo(features=self.config.features	)





							def 				UpperCAmelCase__							(  self  :Any					,							lowerCamelCase_  :List[str]	):





										"""simple docstring"""

										if not self.config.data_files:
													raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
										lowerCamelCase__     :					Any		=dl_manager.download_and_extract(self.config.data_files	)
										if isinstance(lowercase_					,							(str, list, tuple)	):
													lowerCamelCase__     :					Optional[Any]		=data_files
													if isinstance(lowercase_					,							lowercase_	):
																lowerCamelCase__     :					int		=[files]
													lowerCamelCase__     :					Dict		=[dl_manager.iter_files(lowercase_	) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,							gen_kwargs={'files': files}	)]
										lowerCamelCase__     :					Optional[int]		=[]
										for split_name, files in data_files.items():
													if isinstance(lowercase_					,							lowercase_	):
																lowerCamelCase__     :					Optional[int]		=[files]
													lowerCamelCase__     :					Optional[Any]		=[dl_manager.iter_files(lowercase_	) for file in files]
													splits.append(datasets.SplitGenerator(name=lowercase_					,							gen_kwargs={'files': files}	)	)
										return splits





							def 				UpperCAmelCase__							(  self  :Union[str, Any]					,							lowerCamelCase_  :Optional[Any]	):





										"""simple docstring"""

										if self.config.features is not None:
													lowerCamelCase__     :					Optional[int]		=self.config.features.arrow_schema
													if all(not require_storage_cast(lowercase_	) for feature in self.config.features.values()	):
																# cheaper cast
																lowerCamelCase__     :					List[Any]		=pa_table.cast(lowercase_	)
													else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																lowerCamelCase__     :					Optional[Any]		=table_cast(lowercase_					,							lowercase_	)
													return pa_table
										else:
													return pa_table.cast(pa.schema({'text': pa.string()}	)	)





							def 				UpperCAmelCase__							(  self  :Union[str, Any]					,							lowerCamelCase_  :List[str]	):





										"""simple docstring"""

										lowerCamelCase__     :					Union[str, Any]		=list(self.config.features	) if self.config.features is not None else ['text']
										for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase_	)	):
													# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
													with open(lowercase_					,							encoding=self.config.encoding					,							errors=self.config.errors	) as f:
																if self.config.sample_by == "line":
																			lowerCamelCase__     :					Dict		=0
																			while True:
																						lowerCamelCase__     :					str		=f.read(self.config.chunksize	)
																						if not batch:
																									break
																						batch += f.readline()  # finish current line
																						# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																						lowerCamelCase__     :					List[str]		=StringIO(lowercase_	).readlines()
																						if not self.config.keep_linebreaks:
																									lowerCamelCase__     :					Tuple		=[line.rstrip('\n'	) for line in batch]
																						lowerCamelCase__     :					int		=pa.Table.from_arrays([pa.array(lowercase_	)]					,							names=lowercase_	)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(lowercase_	)
																						batch_idx += 1
																elif self.config.sample_by == "paragraph":
																			lowerCamelCase__     :					str		=0
																			lowerCamelCase__     :					Dict		=''
																			while True:
																						lowerCamelCase__     :					Union[str, Any]		=f.read(self.config.chunksize	)
																						if not new_batch:
																									break
																						batch += new_batch
																						batch += f.readline()  # finish current line
																						lowerCamelCase__     :					str		=batch.split('\n\n'	)
																						lowerCamelCase__     :					Union[str, Any]		=pa.Table.from_arrays(
																						    [pa.array([example for example in batch[:-1] if example]	)]					,							names=lowercase_	)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(lowercase_	)
																						batch_idx += 1
																						lowerCamelCase__     :					List[str]		=batch[-1]
																			if batch:
																						lowerCamelCase__     :					str		=pa.Table.from_arrays([pa.array([batch]	)]					,							names=lowercase_	)
																						yield (file_idx, batch_idx), self._cast_table(lowercase_	)
																elif self.config.sample_by == "document":
																			lowerCamelCase__     :					Optional[int]		=f.read()
																			lowerCamelCase__     :					List[Any]		=pa.Table.from_arrays([pa.array([text]	)]					,							names=lowercase_	)
																			yield file_idx, self._cast_table(lowercase_	)

"""simple docstring"""


from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


lowerCAmelCase								=						logging.get_logger(__name__)

lowerCAmelCase								=						{
    """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json""",
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}

class    A_     (					A__       ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_     				=       """blenderbot"""
							SCREAMING_SNAKE_CASE_     				=       ["""past_key_values"""]
							SCREAMING_SNAKE_CASE_     				=       {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}





							def __init__(  self  :Optional[int]					,							lowerCamelCase_  :Optional[Any]=8_008					,							lowerCamelCase_  :Optional[Any]=128					,							lowerCamelCase_  :Optional[int]=2					,							lowerCamelCase_  :Tuple=10_240					,							lowerCamelCase_  :Optional[int]=32					,							lowerCamelCase_  :Optional[int]=24					,							lowerCamelCase_  :Optional[Any]=10_240					,							lowerCamelCase_  :Union[str, Any]=32					,							lowerCamelCase_  :Any=0.0					,							lowerCamelCase_  :int=0.0					,							lowerCamelCase_  :str=True					,							lowerCamelCase_  :List[Any]=True					,							lowerCamelCase_  :Tuple="gelu"					,							lowerCamelCase_  :str=2_560					,							lowerCamelCase_  :Union[str, Any]=0.1					,							lowerCamelCase_  :Optional[int]=0.0					,							lowerCamelCase_  :Union[str, Any]=0.0					,							lowerCamelCase_  :List[str]=0.02					,							lowerCamelCase_  :Any=1					,							lowerCamelCase_  :List[Any]=False					,							lowerCamelCase_  :str=0					,							lowerCamelCase_  :Dict=1					,							lowerCamelCase_  :List[Any]=2					,							lowerCamelCase_  :int=3					,							lowerCamelCase_  :int=2					,							**lowerCamelCase_  :List[str]					,							):





										"""simple docstring"""

										lowerCamelCase__     :					Optional[int]		=vocab_size
										lowerCamelCase__     :					Optional[int]		=max_position_embeddings
										lowerCamelCase__     :					Dict		=d_model
										lowerCamelCase__     :					str		=encoder_ffn_dim
										lowerCamelCase__     :					Optional[Any]		=encoder_layers
										lowerCamelCase__     :					List[Any]		=encoder_attention_heads
										lowerCamelCase__     :					Any		=decoder_ffn_dim
										lowerCamelCase__     :					Any		=decoder_layers
										lowerCamelCase__     :					Any		=decoder_attention_heads
										lowerCamelCase__     :					Dict		=dropout
										lowerCamelCase__     :					Any		=attention_dropout
										lowerCamelCase__     :					Dict		=activation_dropout
										lowerCamelCase__     :					Tuple		=activation_function
										lowerCamelCase__     :					List[Any]		=init_std
										lowerCamelCase__     :					str		=encoder_layerdrop
										lowerCamelCase__     :					Any		=decoder_layerdrop
										lowerCamelCase__     :					Union[str, Any]		=use_cache
										lowerCamelCase__     :					List[str]		=encoder_layers
										lowerCamelCase__     :					Optional[int]		=scale_embedding  # scale factor will be sqrt(d_model) if True

										super().__init__(
										    pad_token_id=lowerCamelCase_					,							bos_token_id=lowerCamelCase_					,							eos_token_id=lowerCamelCase_					,							is_encoder_decoder=lowerCamelCase_					,							decoder_start_token_id=lowerCamelCase_					,							encoder_no_repeat_ngram_size=lowerCamelCase_					,							forced_eos_token_id=lowerCamelCase_					,							**lowerCamelCase_					,							)






class    A_     (					A__       ):



							"""simple docstring"""






							@property
							def 				UpperCAmelCase__							(  self  :Tuple	):





										"""simple docstring"""

										if self.task in ["default", "seq2seq-lm"]:
													lowerCamelCase__     :					Dict		=OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													    ]	)
													if self.use_past:
																lowerCamelCase__     :					Dict		={0: 'batch'}
																lowerCamelCase__     :					Tuple		={0: 'batch', 1: 'past_decoder_sequence + sequence'}
													else:
																lowerCamelCase__     :					Optional[int]		={0: 'batch', 1: 'decoder_sequence'}
																lowerCamelCase__     :					Tuple		={0: 'batch', 1: 'decoder_sequence'}
													if self.use_past:
																self.fill_with_past_key_values_(lowerCamelCase_					,							direction='inputs'	)
										elif self.task == "causal-lm":
													lowerCamelCase__     :					Any		=OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													    ]	)
													if self.use_past:
																lowerCamelCase__	,    lowerCamelCase__     :					List[Any]		=self.num_layers
																for i in range(lowerCamelCase_	):
																			lowerCamelCase__     :					Tuple		={0: 'batch', 2: 'past_sequence + sequence'}
																			lowerCamelCase__     :					Optional[int]		={0: 'batch', 2: 'past_sequence + sequence'}
										else:
													lowerCamelCase__     :					List[str]		=OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													        ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
													        ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
													    ]	)

										return common_inputs





							@property
							# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
							def 				UpperCAmelCase__							(  self  :List[str]	):





										"""simple docstring"""

										if self.task in ["default", "seq2seq-lm"]:
													lowerCamelCase__     :					Tuple		=super().outputs
										else:
													lowerCamelCase__     :					List[Any]		=super(lowerCamelCase_					,							self	).outputs
													if self.use_past:
																lowerCamelCase__	,    lowerCamelCase__     :					List[Any]		=self.num_layers
																for i in range(lowerCamelCase_	):
																			lowerCamelCase__     :					Tuple		={0: 'batch', 2: 'past_sequence + sequence'}
																			lowerCamelCase__     :					str		={0: 'batch', 2: 'past_sequence + sequence'}
										return common_outputs





							def 				UpperCAmelCase__							(  self  :List[Any]					,							lowerCamelCase_  :PreTrainedTokenizer					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :Optional[TensorType] = None					,							):





										"""simple docstring"""

										lowerCamelCase__     :					Optional[int]		=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_	)
										# Generate decoder inputs
										lowerCamelCase__     :					List[str]		=seq_length if not self.use_past else 1
										lowerCamelCase__     :					Optional[Any]		=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_	)
										lowerCamelCase__     :					int		={f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
										lowerCamelCase__     :					int		=dict(**lowerCamelCase_					,							**lowerCamelCase_	)

										if self.use_past:
													if not is_torch_available():
																raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'	)
													else:
																import torch
													lowerCamelCase__	,    lowerCamelCase__     :					List[str]		=common_inputs['input_ids'].shape
													lowerCamelCase__     :					Tuple		=common_inputs['decoder_input_ids'].shape[1]
													lowerCamelCase__	,    lowerCamelCase__     :					int		=self.num_attention_heads
													lowerCamelCase__     :					Optional[Any]		=(
													    batch,
													    num_encoder_attention_heads,
													    encoder_seq_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)
													lowerCamelCase__     :					Any		=decoder_seq_length
													lowerCamelCase__     :					Optional[Any]		=(
													    batch,
													    num_decoder_attention_heads,
													    decoder_past_length,
													    self._config.hidden_size // num_decoder_attention_heads,
													)
													lowerCamelCase__     :					Optional[int]		=torch.cat(
													    [common_inputs['decoder_attention_mask'], torch.ones(lowerCamelCase_					,							lowerCamelCase_	)]					,							dim=1	)
													lowerCamelCase__     :					List[str]		=[]
													lowerCamelCase__	,    lowerCamelCase__     :					Dict		=self.num_layers

													for _ in range(lowerCamelCase_	):
																common_inputs["past_key_values"].append(
																    (
																        torch.zeros(lowerCamelCase_	),
																        torch.zeros(lowerCamelCase_	),
																        torch.zeros(lowerCamelCase_	),
																        torch.zeros(lowerCamelCase_	),
																    )	)
										return common_inputs





							def 				UpperCAmelCase__							(  self  :int					,							lowerCamelCase_  :PreTrainedTokenizer					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :Optional[TensorType] = None					,							):





										"""simple docstring"""

										lowerCamelCase__     :					Union[str, Any]		=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_	)

										if self.use_past:
													if not is_torch_available():
																raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'	)
													else:
																import torch
													lowerCamelCase__	,    lowerCamelCase__     :					Optional[int]		=common_inputs['input_ids'].shape
													lowerCamelCase__     :					List[Any]		=seqlen
													lowerCamelCase__	,    lowerCamelCase__     :					Tuple		=self.num_layers
													lowerCamelCase__	,    lowerCamelCase__     :					List[str]		=self.num_attention_heads
													lowerCamelCase__     :					Optional[int]		=(
													    batch,
													    num_encoder_attention_heads,
													    past_key_values_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)
													lowerCamelCase__     :					List[str]		=common_inputs['attention_mask'].dtype
													lowerCamelCase__     :					Tuple		=torch.cat(
													    [common_inputs['attention_mask'], torch.ones(lowerCamelCase_					,							lowerCamelCase_					,							dtype=lowerCamelCase_	)]					,							dim=1	)
													lowerCamelCase__     :					Optional[int]		=[
													    (torch.zeros(lowerCamelCase_	), torch.zeros(lowerCamelCase_	)) for _ in range(lowerCamelCase_	)
													]
										return common_inputs





							def 				UpperCAmelCase__							(  self  :Union[str, Any]					,							lowerCamelCase_  :PreTrainedTokenizer					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :Optional[TensorType] = None					,							):





										"""simple docstring"""

										lowerCamelCase__     :					List[Any]		=compute_effective_axis_dimension(
										    lowerCamelCase_					,							fixed_dimension=OnnxConfig.default_fixed_batch					,							num_token_to_add=0	)

										# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
										lowerCamelCase__     :					Optional[int]		=tokenizer.num_special_tokens_to_add(lowerCamelCase_	)
										lowerCamelCase__     :					int		=compute_effective_axis_dimension(
										    lowerCamelCase_					,							fixed_dimension=OnnxConfig.default_fixed_sequence					,							num_token_to_add=lowerCamelCase_	)

										# Generate dummy inputs according to compute batch and sequence
										lowerCamelCase__     :					List[Any]		=[' '.join([tokenizer.unk_token]	) * seq_length] * batch_size
										lowerCamelCase__     :					Optional[Any]		=dict(tokenizer(lowerCamelCase_					,							return_tensors=lowerCamelCase_	)	)
										return common_inputs





							def 				UpperCAmelCase__							(  self  :Tuple					,							lowerCamelCase_  :PreTrainedTokenizer					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :int = -1					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :Optional[TensorType] = None					,							):





										"""simple docstring"""

										if self.task in ["default", "seq2seq-lm"]:
													lowerCamelCase__     :					str		=self._generate_dummy_inputs_for_default_and_seqaseq_lm(
													    lowerCamelCase_					,							batch_size=lowerCamelCase_					,							seq_length=lowerCamelCase_					,							is_pair=lowerCamelCase_					,							framework=lowerCamelCase_	)

										elif self.task == "causal-lm":
													lowerCamelCase__     :					List[Any]		=self._generate_dummy_inputs_for_causal_lm(
													    lowerCamelCase_					,							batch_size=lowerCamelCase_					,							seq_length=lowerCamelCase_					,							is_pair=lowerCamelCase_					,							framework=lowerCamelCase_	)
										else:
													lowerCamelCase__     :					Dict		=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
													    lowerCamelCase_					,							batch_size=lowerCamelCase_					,							seq_length=lowerCamelCase_					,							is_pair=lowerCamelCase_					,							framework=lowerCamelCase_	)

										return common_inputs





							def 				UpperCAmelCase__							(  self  :Optional[int]					,							lowerCamelCase_  :Tuple					,							lowerCamelCase_  :Optional[Any]					,							lowerCamelCase_  :List[Any]					,							lowerCamelCase_  :Dict	):





										"""simple docstring"""

										if self.task in ["default", "seq2seq-lm"]:
													lowerCamelCase__     :					List[Any]		=super()._flatten_past_key_values_(lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_	)
										else:
													lowerCamelCase__     :					List[Any]		=super(lowerCamelCase_					,							self	)._flatten_past_key_values_(
													    lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_	)





							def 				UpperCAmelCase__							(  self  :Any					,							lowerCamelCase_  :Mapping[str, Mapping[int, str]]					,							lowerCamelCase_  :str	):





										"""simple docstring"""

										if direction not in ["inputs", "outputs"]:
													raise ValueError(f"""direction must either be \"inputs\" or \"outputs\", but {direction} was given"""	)

										lowerCamelCase__     :					Optional[Any]		='past_key_values' if direction == 'inputs' else 'present'
										lowerCamelCase__	,    lowerCamelCase__     :					Union[str, Any]		=self.num_layers

										lowerCamelCase__     :					Optional[Any]		='past_encoder_sequence'
										lowerCamelCase__     :					Union[str, Any]		='past_decoder_sequence' if direction == 'inputs' else 'past_decoder_sequence + sequence'

										for i in range(lowerCamelCase_	):
													lowerCamelCase__     :					Union[str, Any]		={0: 'batch', 2: decoder_sequence}
													lowerCamelCase__     :					Any		={0: 'batch', 2: decoder_sequence}
													lowerCamelCase__     :					List[Any]		={0: 'batch', 2: encoder_sequence}
													lowerCamelCase__     :					Tuple		={0: 'batch', 2: encoder_sequence}