




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowercase__    :					Any					=						logging.get_logger(__name__)

lowercase__    :					Union[str, Any]					=						{
    'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}
class 		UpperCAmelCase       (    _A      ):







						'''simple docstring'''





						lowerCAmelCase_							=       '''m2m_100'''
						lowerCAmelCase_							=       ['''past_key_values''']
						lowerCAmelCase_							=       {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}






						def __init__(						self							:					Tuple	,     __lowercase							:					int=12_81_12	,     __lowercase							:					str=10_24	,     __lowercase							:					Optional[Any]=12	,     __lowercase							:					Tuple=40_96	,     __lowercase							:					Optional[int]=16	,     __lowercase							:					List[Any]=12	,     __lowercase							:					List[Any]=40_96	,     __lowercase							:					Optional[int]=16	,     __lowercase							:					Optional[int]=0.05	,     __lowercase							:					str=0.05	,     __lowercase							:					Optional[Any]=True	,     __lowercase							:					Dict=True	,     __lowercase							:					List[str]="relu"	,     __lowercase							:					Any=10_24	,     __lowercase							:					str=0.1	,     __lowercase							:					Optional[int]=0.1	,     __lowercase							:					Any=0.0	,     __lowercase							:					Any=0.02	,     __lowercase							:					Optional[Any]=2	,     __lowercase							:					List[Any]=True	,     __lowercase							:					List[str]=1	,     __lowercase							:					str=0	,     __lowercase							:					Any=2	,     **__lowercase							:					Tuple	,     ):






							"""simple docstring"""






							snake_case_		   =					vocab_size
							snake_case_		   =					max_position_embeddings
							snake_case_		   =					d_model
							snake_case_		   =					encoder_ffn_dim
							snake_case_		   =					encoder_layers
							snake_case_		   =					encoder_attention_heads
							snake_case_		   =					decoder_ffn_dim
							snake_case_		   =					decoder_layers
							snake_case_		   =					decoder_attention_heads
							snake_case_		   =					dropout
							snake_case_		   =					attention_dropout
							snake_case_		   =					activation_dropout
							snake_case_		   =					activation_function
							snake_case_		   =					init_std
							snake_case_		   =					encoder_layerdrop
							snake_case_		   =					decoder_layerdrop
							snake_case_		   =					use_cache
							snake_case_		   =					encoder_layers
							snake_case_		   =					scale_embedding  # scale factor will be sqrt(d_model) if True

							super().__init__(
							    pad_token_id=UpperCamelCase__	,     bos_token_id=UpperCamelCase__	,     eos_token_id=UpperCamelCase__	,     is_encoder_decoder=UpperCamelCase__	,     decoder_start_token_id=UpperCamelCase__	,     **UpperCamelCase__	,     )







class 		UpperCAmelCase       (    _A      ):







						'''simple docstring'''





						@property
						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					OrderedDict(
							    [
							        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
							        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
							    ]						)

							if self.use_past:
								snake_case_		   =					{0: "batch"}
								snake_case_		   =					{0: "batch", 1: "past_decoder_sequence + sequence"}
							else:
								snake_case_		   =					{0: "batch", 1: "decoder_sequence"}
								snake_case_		   =					{0: "batch", 1: "decoder_sequence"}

							if self.use_past:
								self.fill_with_past_key_values_(UpperCamelCase__	,     direction="inputs"						)
							return common_inputs






						def 	snake_case__						(						self							:					str	,     __lowercase							:					PreTrainedTokenizer	,     __lowercase							:					int = -1	,     __lowercase							:					int = -1	,     __lowercase							:					bool = False	,     __lowercase							:					Optional[TensorType] = None	,     ):






							"""simple docstring"""






							snake_case_		   =					compute_effective_axis_dimension(
							    UpperCamelCase__	,     fixed_dimension=OnnxConfig.default_fixed_batch	,     num_token_to_add=0						)

							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							snake_case_		   =					tokenizer.num_special_tokens_to_add(UpperCamelCase__						)
							snake_case_		   =					compute_effective_axis_dimension(
							    UpperCamelCase__	,     fixed_dimension=OnnxConfig.default_fixed_sequence	,     num_token_to_add=UpperCamelCase__						)

							# Generate dummy inputs according to compute batch and sequence
							snake_case_		   =					[" ".join([tokenizer.unk_token]						) * seq_length] * batch_size
							snake_case_		   =					dict(tokenizer(UpperCamelCase__	,     return_tensors=UpperCamelCase__						)						)
							return common_inputs






						def 	snake_case__						(						self							:					List[Any]	,     __lowercase							:					PreTrainedTokenizer	,     __lowercase							:					int = -1	,     __lowercase							:					int = -1	,     __lowercase							:					bool = False	,     __lowercase							:					Optional[TensorType] = None	,     ):






							"""simple docstring"""






							snake_case_		   =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__						)

							# Generate decoder inputs
							snake_case_		   =					seq_length if not self.use_past else 1
							snake_case_		   =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__	,     UpperCamelCase__						)
							snake_case_		   =					{f"decoder_{name}": tensor for name, tensor in decoder_inputs.items()}
							snake_case_		   =					dict(**UpperCamelCase__	,     **UpperCamelCase__						)

							if self.use_past:
								if not is_torch_available():
									raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."						)
								else:
									import torch
								snake_case_   ,				snake_case_		   =					common_inputs["input_ids"].shape
								snake_case_		   =					common_inputs["decoder_input_ids"].shape[1]
								snake_case_   ,				snake_case_		   =					self.num_attention_heads
								snake_case_		   =					(
								    batch,
								    num_encoder_attention_heads,
								    encoder_seq_length,
								    self._config.hidden_size // num_encoder_attention_heads,
								)
								snake_case_		   =					decoder_seq_length + 3
								snake_case_		   =					(
								    batch,
								    num_decoder_attention_heads,
								    decoder_past_length,
								    self._config.hidden_size // num_decoder_attention_heads,
								)

								snake_case_		   =					torch.cat(
								    [common_inputs["decoder_attention_mask"], torch.ones(UpperCamelCase__	,     UpperCamelCase__						)]	,     dim=1						)

								snake_case_		   =					[]
								# If the number of encoder and decoder layers are present in the model configuration, both are considered
								snake_case_   ,				snake_case_		   =					self.num_layers
								snake_case_		   =					min(UpperCamelCase__	,     UpperCamelCase__						)
								snake_case_		   =					max(UpperCamelCase__	,     UpperCamelCase__						) - min_num_layers
								snake_case_		   =					"encoder" if num_encoder_layers > num_decoder_layers else "decoder"

								for _ in range(UpperCamelCase__						):
									common_inputs["past_key_values"].append(
									    (
									        torch.zeros(UpperCamelCase__						),
									        torch.zeros(UpperCamelCase__						),
									        torch.zeros(UpperCamelCase__						),
									        torch.zeros(UpperCamelCase__						),
									    )						)
								# TODO: test this.
								snake_case_		   =					encoder_shape if remaining_side_name == "encoder" else decoder_shape
								for _ in range(UpperCamelCase__	,     UpperCamelCase__						):
									common_inputs["past_key_values"].append((torch.zeros(UpperCamelCase__						), torch.zeros(UpperCamelCase__						))						)
							return common_inputs
						lowerCAmelCase_							=       _generate_dummy_inputs_for_default_and_seqaseq_lm






import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowercase__    :					str					=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 		UpperCAmelCase       (    datasets.BuilderConfig      ):







						'''simple docstring'''





						lowerCAmelCase_							=       None







class 		UpperCAmelCase       (    datasets.ArrowBasedBuilder      ):







						'''simple docstring'''





						lowerCAmelCase_							=       ArrowConfig






						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							return datasets.DatasetInfo(features=self.config.features						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Union[str, Any]						):






							"""simple docstring"""






							if not self.config.data_files:
								raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"						)
							snake_case_		   =					dl_manager.download_and_extract(self.config.data_files						)
							if isinstance(__lowercase	,     (str, list, tuple)						):
								snake_case_		   =					data_files
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,     gen_kwargs={"files": files}						)]
							snake_case_		   =					[]
							for split_name, files in data_files.items():
								if isinstance(__lowercase	,     __lowercase						):
									snake_case_		   =					[files]
								# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
								snake_case_		   =					[dl_manager.iter_files(__lowercase						) for file in files]
								# Infer features is they are stoed in the arrow schema
								if self.info.features is None:
									for file in itertools.chain.from_iterable(__lowercase						):
										with open(__lowercase	,     "rb"						) as f:
											snake_case_		   =					datasets.Features.from_arrow_schema(pa.ipc.open_stream(__lowercase						).schema						)
										break
								splits.append(datasets.SplitGenerator(name=__lowercase	,     gen_kwargs={"files": files}						)						)
							return splits






						def 	snake_case__						(						self							:					str	,     __lowercase							:					pa.Table						):






							"""simple docstring"""






							if self.info.features is not None:
								# more expensive cast to support nested features with keys in a different order
								# allows str <-> int/float or str to Audio for example
								snake_case_		   =					table_cast(__lowercase	,     self.info.features.arrow_schema						)
							return pa_table






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[Any]						):






							"""simple docstring"""






							for file_idx, file in enumerate(itertools.chain.from_iterable(__lowercase						)						):
								with open(__lowercase	,     "rb"						) as f:
									try:
										for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__lowercase						)						):
											snake_case_		   =					pa.Table.from_batches([record_batch]						)
											# Uncomment for debugging (will print the Arrow table size and elements)
											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield f"{file_idx}_{batch_idx}", self._cast_table(__lowercase						)
									except ValueError as e:
										logger.error(f"Failed to read file '{file}' with error {type(__lowercase						)}: {e}"						)
										raise
