




import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode




def       __magic_name__    (			A		: Union[str, Any],  **A		: Optional[int]       ):






			'''simple docstring'''







			return RunBeamCommand(
			    args.dataset,  args.name,  args.cache_dir,  args.beam_pipeline_options,  args.data_dir,  args.all_configs,  args.save_info or args.save_infos,  args.ignore_verifications,  args.force_redownload,  **A,  )






class    snake_case__							(_UpperCamelCase       ):
				"""simple docstring"""



				@staticmethod
				def 							__UpperCAmelCase							(				__lowerCamelCase    :					ArgumentParser							) ->					str:
							a   =      parser.add_parser("run_beam" ,				help="Run a Beam dataset processing pipeline"							)
							run_beam_parser.add_argument("dataset" ,				type=__lowerCamelCase ,				help="Name of the dataset to download"							)
							run_beam_parser.add_argument("--name" ,				type=__lowerCamelCase ,				default=__lowerCamelCase ,				help="Dataset config name"							)
							run_beam_parser.add_argument(
							    "--cache_dir" ,				type=__lowerCamelCase ,				default=__lowerCamelCase ,				help="Cache directory where the datasets are stored" ,				)
							run_beam_parser.add_argument(
							    "--beam_pipeline_options" ,				type=__lowerCamelCase ,				default="" ,				help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`" ,				)
							run_beam_parser.add_argument(
							    "--data_dir" ,				type=__lowerCamelCase ,				default=__lowerCamelCase ,				help="Can be used to specify a manual directory to get the files from" ,				)
							run_beam_parser.add_argument("--all_configs" ,				action="store_true" ,				help="Test all dataset configurations"							)
							run_beam_parser.add_argument("--save_info" ,				action="store_true" ,				help="Save the dataset infos file"							)
							run_beam_parser.add_argument(
							    "--ignore_verifications" ,				action="store_true" ,				help="Run the test without checksums and splits checks"							)
							run_beam_parser.add_argument("--force_redownload" ,				action="store_true" ,				help="Force dataset redownload"							)
							# aliases
							run_beam_parser.add_argument("--save_infos" ,				action="store_true" ,				help="alias for save_info"							)
							run_beam_parser.set_defaults(func=__lowerCamelCase							)





				def __init__(				self    :					List[str] ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					bool ,				__lowerCamelCase    :					bool ,				__lowerCamelCase    :					bool ,				__lowerCamelCase    :					bool ,				**__lowerCamelCase    :					Tuple ,				) ->					Optional[Any]:
							a   =      dataset
							a   =      name
							a   =      cache_dir
							a   =      beam_pipeline_options
							a   =      data_dir
							a   =      all_configs
							a   =      save_infos
							a   =      ignore_verifications
							a   =      force_redownload
							a   =      config_kwargs





				def 							__UpperCAmelCase							(				self    :					Tuple							) ->					Optional[int]:
							import apache_beam as beam

							if self._name is not None and self._all_configs:
										print("Both parameters `name` and `all_configs` can't be used at once."							)
										exit(1							)
							a   ,		a   =      self._dataset, self._name
							a   =      dataset_module_factory(__lowerCamelCase							)
							a   =      import_main_class(dataset_module.module_path							)
							a   =      []
							if self._beam_pipeline_options:
										a   =      beam.options.pipeline_options.PipelineOptions(
										    flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(","							) if opt]							)
							else:
										a   =      None
							if self._all_configs and len(builder_cls.BUILDER_CONFIGS							) > 0:
										for builder_config in builder_cls.BUILDER_CONFIGS:
													builders.append(
													    builder_cls(
													        config_name=builder_config.name ,				data_dir=self._data_dir ,				hash=dataset_module.hash ,				beam_options=__lowerCamelCase ,				cache_dir=self._cache_dir ,				base_path=dataset_module.builder_kwargs.get("base_path"							) ,				)							)
							else:
										builders.append(
										    builder_cls(
										        config_name=__lowerCamelCase ,				data_dir=self._data_dir ,				beam_options=__lowerCamelCase ,				cache_dir=self._cache_dir ,				base_path=dataset_module.builder_kwargs.get("base_path"							) ,				**self._config_kwargs ,				)							)

							for builder in builders:
										builder.download_and_prepare(
										    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
										    if not self._force_redownload
										    else DownloadMode.FORCE_REDOWNLOAD ,				download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH							) ,				verification_mode=VerificationMode.NO_CHECKS
										    if self._ignore_verifications
										    else VerificationMode.ALL_CHECKS ,				try_from_hf_gcs=__lowerCamelCase ,				)
										if self._save_infos:
													builder._save_infos()

							print("Apache beam run successful."							)

							# If save_infos=True, the dataset infos file is created next to the loaded module file.
							# Let's move it to the original directory of the dataset script, to allow the user to
							# upload them on S3 at the same time afterwards.
							if self._save_infos:
										a   =      os.path.join(builder_cls.get_imported_module_dir() ,				config.DATASETDICT_INFOS_FILENAME							)

										a   =      Path(__lowerCamelCase							).name + ".py"

										a   =      os.path.join(__lowerCamelCase ,				__lowerCamelCase							)
										if os.path.isfile(__lowerCamelCase							):
													a   =      os.path.dirname(__lowerCamelCase							)
										elif os.path.isfile(__lowerCamelCase							):
													a   =      path
										else:  # in case of a remote dataset
													print(f"""Dataset Infos file saved at {dataset_infos_path}"""							)
													exit(1							)

										# Move datasetinfo back to the user
										a   =      os.path.join(__lowerCamelCase ,				config.DATASETDICT_INFOS_FILENAME							)
										copyfile(__lowerCamelCase ,				__lowerCamelCase							)
										print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""							)












from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__lowerCAmelCase		:    Dict	      =		logging.get_logger(__name__)

__lowerCAmelCase		:    Optional[int]	      =		{
    'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}




class    snake_case__							(_UpperCamelCase       ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_	:							int										= """mbart"""
				SCREAMING_SNAKE_CASE_	:							Optional[Any]										= ["""past_key_values"""]
				SCREAMING_SNAKE_CASE_	:							Any										= {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}





				def __init__(				self    :					List[Any] ,				__lowerCamelCase    :					str=5_02_65 ,				__lowerCamelCase    :					List[Any]=10_24 ,				__lowerCamelCase    :					Optional[Any]=12 ,				__lowerCamelCase    :					List[Any]=40_96 ,				__lowerCamelCase    :					int=16 ,				__lowerCamelCase    :					List[Any]=12 ,				__lowerCamelCase    :					Tuple=40_96 ,				__lowerCamelCase    :					Tuple=16 ,				__lowerCamelCase    :					Optional[Any]=0.0 ,				__lowerCamelCase    :					int=0.0 ,				__lowerCamelCase    :					Any=True ,				__lowerCamelCase    :					List[Any]=True ,				__lowerCamelCase    :					str="gelu" ,				__lowerCamelCase    :					Tuple=10_24 ,				__lowerCamelCase    :					Any=0.1 ,				__lowerCamelCase    :					Dict=0.0 ,				__lowerCamelCase    :					Tuple=0.0 ,				__lowerCamelCase    :					Optional[int]=0.02 ,				__lowerCamelCase    :					Dict=0.0 ,				__lowerCamelCase    :					Union[str, Any]=False ,				__lowerCamelCase    :					Optional[int]=1 ,				__lowerCamelCase    :					Tuple=0 ,				__lowerCamelCase    :					List[Any]=2 ,				__lowerCamelCase    :					Dict=2 ,				**__lowerCamelCase    :					Optional[int] ,				) ->					Optional[int]:
							a   =      vocab_size
							a   =      max_position_embeddings
							a   =      d_model
							a   =      encoder_ffn_dim
							a   =      encoder_layers
							a   =      encoder_attention_heads
							a   =      decoder_ffn_dim
							a   =      decoder_layers
							a   =      decoder_attention_heads
							a   =      dropout
							a   =      attention_dropout
							a   =      activation_dropout
							a   =      activation_function
							a   =      init_std
							a   =      encoder_layerdrop
							a   =      decoder_layerdrop
							a   =      classifier_dropout
							a   =      use_cache
							a   =      encoder_layers
							a   =      scale_embedding  # scale factor will be sqrt(d_model) if True
							super().__init__(
							    pad_token_id=__lowerCamelCase ,				bos_token_id=__lowerCamelCase ,				eos_token_id=__lowerCamelCase ,				is_encoder_decoder=__lowerCamelCase ,				forced_eos_token_id=__lowerCamelCase ,				**__lowerCamelCase ,				)




class    snake_case__							(_UpperCamelCase       ):
				"""simple docstring"""



				@property
				def 							__UpperCAmelCase							(				self    :					Optional[Any]							) ->					Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
										a   =      OrderedDict(
										    [
										        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
										        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
										    ]							)

										if self.use_past:
													a   =      {0: "batch"}
													a   =      {0: "batch", 1: "past_decoder_sequence + sequence"}
										else:
													a   =      {0: "batch", 1: "decoder_sequence"}
													a   =      {0: "batch", 1: "decoder_sequence"}

										if self.use_past:
													self.fill_with_past_key_values_(__lowerCamelCase ,				direction="inputs"							)
							elif self.task == "causal-lm":
										# TODO: figure this case out.
										a   =      OrderedDict(
										    [
										        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
										        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
										    ]							)
										if self.use_past:
													a   ,		a   =      self.num_layers
													for i in range(__lowerCamelCase							):
																a   =      {0: "batch", 2: "past_sequence + sequence"}
																a   =      {0: "batch", 2: "past_sequence + sequence"}
							else:
										a   =      OrderedDict(
										    [
										        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
										        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
										        ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
										        ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
										    ]							)

							return common_inputs





				@property
				def 							__UpperCAmelCase							(				self    :					Any							) ->					Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
										a   =      super().outputs
							else:
										a   =      super(__lowerCamelCase ,				self							).outputs
										if self.use_past:
													a   ,		a   =      self.num_layers
													for i in range(__lowerCamelCase							):
																a   =      {0: "batch", 2: "past_sequence + sequence"}
																a   =      {0: "batch", 2: "past_sequence + sequence"}
							return common_outputs





				def 							__UpperCAmelCase							(				self    :					Union[str, Any] ,				__lowerCamelCase    :					PreTrainedTokenizer ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					bool = False ,				__lowerCamelCase    :					Optional[TensorType] = None ,				) ->					Mapping[str, Any]:
							a   =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    __lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase							)

							# Generate decoder inputs
							a   =      seq_length if not self.use_past else 1
							a   =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    __lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase							)
							a   =      {f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
							a   =      dict(**__lowerCamelCase ,				**__lowerCamelCase							)

							if self.use_past:
										if not is_torch_available():
													raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."							)
										else:
													import torch
										a   ,		a   =      common_inputs["input_ids"].shape
										a   =      common_inputs["decoder_input_ids"].shape[1]
										a   ,		a   =      self.num_attention_heads
										a   =      (
										    batch,
										    num_encoder_attention_heads,
										    encoder_seq_length,
										    self._config.hidden_size // num_encoder_attention_heads,
										)
										a   =      decoder_seq_length + 3
										a   =      (
										    batch,
										    num_decoder_attention_heads,
										    decoder_past_length,
										    self._config.hidden_size // num_decoder_attention_heads,
										)

										a   =      torch.cat(
										    [common_inputs["decoder_attention_mask"], torch.ones(__lowerCamelCase ,				__lowerCamelCase							)] ,				dim=1							)

										a   =      []
										# If the number of encoder and decoder layers are present in the model configuration, both are considered
										a   ,		a   =      self.num_layers
										a   =      min(__lowerCamelCase ,				__lowerCamelCase							)
										a   =      max(__lowerCamelCase ,				__lowerCamelCase							) - min_num_layers
										a   =      "encoder" if num_encoder_layers > num_decoder_layers else "decoder"

										for _ in range(__lowerCamelCase							):
													common_inputs["past_key_values"].append(
													    (
													        torch.zeros(__lowerCamelCase							),
													        torch.zeros(__lowerCamelCase							),
													        torch.zeros(__lowerCamelCase							),
													        torch.zeros(__lowerCamelCase							),
													    )							)
										# TODO: test this.
										a   =      encoder_shape if remaining_side_name == "encoder" else decoder_shape
										for _ in range(__lowerCamelCase ,				__lowerCamelCase							):
													common_inputs["past_key_values"].append((torch.zeros(__lowerCamelCase							), torch.zeros(__lowerCamelCase							))							)
							return common_inputs





				def 							__UpperCAmelCase							(				self    :					str ,				__lowerCamelCase    :					PreTrainedTokenizer ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					bool = False ,				__lowerCamelCase    :					Optional[TensorType] = None ,				) ->					Mapping[str, Any]:
							a   =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    __lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase							)

							if self.use_past:
										if not is_torch_available():
													raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."							)
										else:
													import torch
										a   ,		a   =      common_inputs["input_ids"].shape
										# Not using the same length for past_key_values
										a   =      seqlen + 2
										a   ,		a   =      self.num_layers
										a   ,		a   =      self.num_attention_heads
										a   =      (
										    batch,
										    num_encoder_attention_heads,
										    past_key_values_length,
										    self._config.hidden_size // num_encoder_attention_heads,
										)

										a   =      common_inputs["attention_mask"].dtype
										a   =      torch.cat(
										    [common_inputs["attention_mask"], torch.ones(__lowerCamelCase ,				__lowerCamelCase ,				dtype=__lowerCamelCase							)] ,				dim=1							)
										a   =      [
										    (torch.zeros(__lowerCamelCase							), torch.zeros(__lowerCamelCase							)) for _ in range(__lowerCamelCase							)
										]
							return common_inputs





				def 							__UpperCAmelCase							(				self    :					Union[str, Any] ,				__lowerCamelCase    :					PreTrainedTokenizer ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					bool = False ,				__lowerCamelCase    :					Optional[TensorType] = None ,				) ->					Mapping[str, Any]:
							# Copied from OnnxConfig.generate_dummy_inputs
							# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
							a   =      compute_effective_axis_dimension(
							    __lowerCamelCase ,				fixed_dimension=OnnxConfig.default_fixed_batch ,				num_token_to_add=0							)

							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							a   =      tokenizer.num_special_tokens_to_add(__lowerCamelCase							)
							a   =      compute_effective_axis_dimension(
							    __lowerCamelCase ,				fixed_dimension=OnnxConfig.default_fixed_sequence ,				num_token_to_add=__lowerCamelCase							)

							# Generate dummy inputs according to compute batch and sequence
							a   =      [" ".join([tokenizer.unk_token]							) * seq_length] * batch_size
							a   =      dict(tokenizer(__lowerCamelCase ,				return_tensors=__lowerCamelCase							)							)
							return common_inputs





				def 							__UpperCAmelCase							(				self    :					str ,				__lowerCamelCase    :					PreTrainedTokenizer ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					int = -1 ,				__lowerCamelCase    :					bool = False ,				__lowerCamelCase    :					Optional[TensorType] = None ,				) ->					Mapping[str, Any]:
							if self.task in ["default", "seq2seq-lm"]:
										a   =      self._generate_dummy_inputs_for_default_and_seqaseq_lm(
										    __lowerCamelCase ,				batch_size=__lowerCamelCase ,				seq_length=__lowerCamelCase ,				is_pair=__lowerCamelCase ,				framework=__lowerCamelCase							)

							elif self.task == "causal-lm":
										a   =      self._generate_dummy_inputs_for_causal_lm(
										    __lowerCamelCase ,				batch_size=__lowerCamelCase ,				seq_length=__lowerCamelCase ,				is_pair=__lowerCamelCase ,				framework=__lowerCamelCase							)
							else:
										a   =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    __lowerCamelCase ,				batch_size=__lowerCamelCase ,				seq_length=__lowerCamelCase ,				is_pair=__lowerCamelCase ,				framework=__lowerCamelCase							)

							return common_inputs





				def 							__UpperCAmelCase							(				self    :					Optional[int] ,				__lowerCamelCase    :					Union[str, Any] ,				__lowerCamelCase    :					int ,				__lowerCamelCase    :					str ,				__lowerCamelCase    :					Optional[Any]							) ->					Dict:
							if self.task in ["default", "seq2seq-lm"]:
										a   =      super()._flatten_past_key_values_(__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase							)
							else:
										a   =      super(__lowerCamelCase ,				self							)._flatten_past_key_values_(
										    __lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase							)






