





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode







def  lowerCAmelCase__(		lowercase    :      Dict				,       **lowercase    :      Optional[int]			)	->      Tuple:
  return RunBeamCommand(
      args.dataset				,       args.name				,       args.cache_dir				,       args.beam_pipeline_options				,       args.data_dir				,       args.all_configs				,       args.save_info or args.save_infos				,       args.ignore_verifications				,       args.force_redownload				,       **_A				,       )




class 			_lowerCamelCase	(      lowerCamelCase__      ):



      """simple docstring"""






      @staticmethod
      def   UpperCAmelCase						(	UpperCAmelCase				)    ->				Tuple:






        '''simple docstring'''






        __snake_case				:			List[Any]            =    parser.add_parser("run_beam"			,  help="Run a Beam dataset processing pipeline"				)
        run_beam_parser.add_argument("dataset"			,  type=lowercase_			,  help="Name of the dataset to download"				)
        run_beam_parser.add_argument("--name"			,  type=lowercase_			,  default=lowercase_			,  help="Dataset config name"				)
        run_beam_parser.add_argument(
            "--cache_dir"			,  type=lowercase_			,  default=lowercase_			,  help="Cache directory where the datasets are stored"			,  )
        run_beam_parser.add_argument(
            "--beam_pipeline_options"			,  type=lowercase_			,  default=""			,  help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"			,  )
        run_beam_parser.add_argument(
            "--data_dir"			,  type=lowercase_			,  default=lowercase_			,  help="Can be used to specify a manual directory to get the files from"			,  )
        run_beam_parser.add_argument("--all_configs"			,  action="store_true"			,  help="Test all dataset configurations"				)
        run_beam_parser.add_argument("--save_info"			,  action="store_true"			,  help="Save the dataset infos file"				)
        run_beam_parser.add_argument(
            "--ignore_verifications"			,  action="store_true"			,  help="Run the test without checksums and splits checks"				)
        run_beam_parser.add_argument("--force_redownload"			,  action="store_true"			,  help="Force dataset redownload"				)
        # aliases
        run_beam_parser.add_argument("--save_infos"			,  action="store_true"			,  help="alias for save_info"				)
        run_beam_parser.set_defaults(func=lowercase_				)







      def __init__(	self			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  **UpperCAmelCase			,  )    ->				Optional[int]:






        '''simple docstring'''






        __snake_case				:			Optional[Any]            =    dataset
        __snake_case				:			Optional[int]            =    name
        __snake_case				:			Optional[Any]            =    cache_dir
        __snake_case				:			List[str]            =    beam_pipeline_options
        __snake_case				:			Optional[int]            =    data_dir
        __snake_case				:			List[str]            =    all_configs
        __snake_case				:			Dict            =    save_infos
        __snake_case				:			str            =    ignore_verifications
        __snake_case				:			int            =    force_redownload
        __snake_case				:			Tuple            =    config_kwargs







      def   UpperCAmelCase						(	self				)    ->				Optional[int]:






        '''simple docstring'''






        import apache_beam as beam

        if self._name is not None and self._all_configs:
          print("Both parameters `name` and `all_configs` can\'t be used at once."				)
          exit(1				)
        __snake_case      ,				__snake_case				:			Dict            =    self._dataset, self._name
        __snake_case				:			List[Any]            =    dataset_module_factory(lowercase_				)
        __snake_case				:			List[str]            =    import_main_class(dataset_module.module_path				)
        __snake_case				:			Optional[int]            =    []
        if self._beam_pipeline_options:
          __snake_case				:			Any            =    beam.options.pipeline_options.PipelineOptions(
              flags=[F"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(","				) if opt]				)
        else:
          __snake_case				:			Optional[Any]            =    None
        if self._all_configs and len(builder_cls.BUILDER_CONFIGS				) > 0:
          for builder_config in builder_cls.BUILDER_CONFIGS:
            builders.append(
                builder_cls(
                    config_name=builder_config.name			,  data_dir=self._data_dir			,  hash=dataset_module.hash			,  beam_options=lowercase_			,  cache_dir=self._cache_dir			,  base_path=dataset_module.builder_kwargs.get("base_path"				)			,  )				)
        else:
          builders.append(
              builder_cls(
                  config_name=lowercase_			,  data_dir=self._data_dir			,  beam_options=lowercase_			,  cache_dir=self._cache_dir			,  base_path=dataset_module.builder_kwargs.get("base_path"				)			,  **self._config_kwargs			,  )				)

        for builder in builders:
          builder.download_and_prepare(
              download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
              if not self._force_redownload
              else DownloadMode.FORCE_REDOWNLOAD			,  download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH				)			,  verification_mode=VerificationMode.NO_CHECKS
              if self._ignore_verifications
              else VerificationMode.ALL_CHECKS			,  try_from_hf_gcs=lowercase_			,  )
          if self._save_infos:
            builder._save_infos()

        print("Apache beam run successful."				)

        # If save_infos=True, the dataset infos file is created next to the loaded module file.
        # Let's move it to the original directory of the dataset script, to allow the user to
        # upload them on S3 at the same time afterwards.
        if self._save_infos:
          __snake_case				:			Union[str, Any]            =    os.path.join(builder_cls.get_imported_module_dir()			,  config.DATASETDICT_INFOS_FILENAME				)

          __snake_case				:			int            =    Path(lowercase_				).name + ".py"

          __snake_case				:			Tuple            =    os.path.join(lowercase_			,  lowercase_				)
          if os.path.isfile(lowercase_				):
            __snake_case				:			Tuple            =    os.path.dirname(lowercase_				)
          elif os.path.isfile(lowercase_				):
            __snake_case				:			int            =    path
          else:  # in case of a remote dataset
            print(F"""Dataset Infos file saved at {dataset_infos_path}"""				)
            exit(1				)

          # Move datasetinfo back to the user
          __snake_case				:			List[Any]            =    os.path.join(lowercase_			,  config.DATASETDICT_INFOS_FILENAME				)
          copyfile(lowercase_			,  lowercase_				)
          print(F"""Dataset Infos file saved at {user_dataset_infos_path}"""				)








from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


_UpperCamelCase    =	logging.get_logger(__name__)

_UpperCamelCase    =	{
    '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json''',
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}




class 			_lowerCamelCase	(      a      ):



      """simple docstring"""






      UpperCAmelCase_  : Tuple				      ="blenderbot"
      UpperCAmelCase_  : Any				      =["past_key_values"]
      UpperCAmelCase_  : Optional[int]				      ={"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}







      def __init__(	self			,  UpperCAmelCase=8008			,  UpperCAmelCase=128			,  UpperCAmelCase=2			,  UpperCAmelCase=10240			,  UpperCAmelCase=32			,  UpperCAmelCase=24			,  UpperCAmelCase=10240			,  UpperCAmelCase=32			,  UpperCAmelCase=0.0			,  UpperCAmelCase=0.0			,  UpperCAmelCase=True			,  UpperCAmelCase=True			,  UpperCAmelCase="gelu"			,  UpperCAmelCase=2560			,  UpperCAmelCase=0.1			,  UpperCAmelCase=0.0			,  UpperCAmelCase=0.0			,  UpperCAmelCase=0.02			,  UpperCAmelCase=1			,  UpperCAmelCase=False			,  UpperCAmelCase=0			,  UpperCAmelCase=1			,  UpperCAmelCase=2			,  UpperCAmelCase=3			,  UpperCAmelCase=2			,  **UpperCAmelCase			,  )    ->				List[Any]:






        '''simple docstring'''






        __snake_case				:			List[Any]            =    vocab_size
        __snake_case				:			List[str]            =    max_position_embeddings
        __snake_case				:			str            =    d_model
        __snake_case				:			int            =    encoder_ffn_dim
        __snake_case				:			Dict            =    encoder_layers
        __snake_case				:			Any            =    encoder_attention_heads
        __snake_case				:			Optional[int]            =    decoder_ffn_dim
        __snake_case				:			int            =    decoder_layers
        __snake_case				:			Dict            =    decoder_attention_heads
        __snake_case				:			Optional[int]            =    dropout
        __snake_case				:			Optional[Any]            =    attention_dropout
        __snake_case				:			Dict            =    activation_dropout
        __snake_case				:			Any            =    activation_function
        __snake_case				:			List[Any]            =    init_std
        __snake_case				:			Any            =    encoder_layerdrop
        __snake_case				:			List[Any]            =    decoder_layerdrop
        __snake_case				:			Any            =    use_cache
        __snake_case				:			Optional[int]            =    encoder_layers
        __snake_case				:			Optional[int]            =    scale_embedding  # scale factor will be sqrt(d_model) if True

        super().__init__(
            pad_token_id=UpperCAmelCase			,  bos_token_id=UpperCAmelCase			,  eos_token_id=UpperCAmelCase			,  is_encoder_decoder=UpperCAmelCase			,  decoder_start_token_id=UpperCAmelCase			,  encoder_no_repeat_ngram_size=UpperCAmelCase			,  forced_eos_token_id=UpperCAmelCase			,  **UpperCAmelCase			,  )




class 			_lowerCamelCase	(      a      ):



      """simple docstring"""






      @property
      def   UpperCAmelCase						(	self				)    ->				Mapping[str, Mapping[int, str]]:






        '''simple docstring'''






        if self.task in ["default", "seq2seq-lm"]:
          __snake_case				:			Any            =    OrderedDict(
              [
                  ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                  ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
              ]				)
          if self.use_past:
            __snake_case				:			List[str]            =    {0: "batch"}
            __snake_case				:			Optional[Any]            =    {0: "batch", 1: "past_decoder_sequence + sequence"}
          else:
            __snake_case				:			int            =    {0: "batch", 1: "decoder_sequence"}
            __snake_case				:			Optional[int]            =    {0: "batch", 1: "decoder_sequence"}
          if self.use_past:
            self.fill_with_past_key_values_(UpperCAmelCase			,  direction="inputs"				)
        elif self.task == "causal-lm":
          __snake_case				:			List[str]            =    OrderedDict(
              [
                  ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                  ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
              ]				)
          if self.use_past:
            __snake_case      ,				__snake_case				:			List[Any]            =    self.num_layers
            for i in range(UpperCAmelCase				):
              __snake_case				:			int            =    {0: "batch", 2: "past_sequence + sequence"}
              __snake_case				:			Tuple            =    {0: "batch", 2: "past_sequence + sequence"}
        else:
          __snake_case				:			Dict            =    OrderedDict(
              [
                  ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                  ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                  ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
                  ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
              ]				)

        return common_inputs







      @property
      # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
      def   UpperCAmelCase						(	self				)    ->				Mapping[str, Mapping[int, str]]:






        '''simple docstring'''






        if self.task in ["default", "seq2seq-lm"]:
          __snake_case				:			Union[str, Any]            =    super().outputs
        else:
          __snake_case				:			Dict            =    super(UpperCAmelCase			,  self				).outputs
          if self.use_past:
            __snake_case      ,				__snake_case				:			Any            =    self.num_layers
            for i in range(UpperCAmelCase				):
              __snake_case				:			Optional[int]            =    {0: "batch", 2: "past_sequence + sequence"}
              __snake_case				:			Tuple            =    {0: "batch", 2: "past_sequence + sequence"}
        return common_outputs







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase = -1			,  UpperCAmelCase = -1			,  UpperCAmelCase = False			,  UpperCAmelCase = None			,  )    ->				Mapping[str, Any]:






        '''simple docstring'''






        __snake_case				:			int            =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)
        # Generate decoder inputs
        __snake_case				:			List[Any]            =    seq_length if not self.use_past else 1
        __snake_case				:			Tuple            =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)
        __snake_case				:			Tuple            =    {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
        __snake_case				:			Dict            =    dict(**UpperCAmelCase			,  **UpperCAmelCase				)

        if self.use_past:
          if not is_torch_available():
            raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
          else:
            import torch
          __snake_case      ,				__snake_case				:			int            =    common_inputs["input_ids"].shape
          __snake_case				:			Union[str, Any]            =    common_inputs["decoder_input_ids"].shape[1]
          __snake_case      ,				__snake_case				:			Optional[Any]            =    self.num_attention_heads
          __snake_case				:			Tuple            =    (
              batch,
              num_encoder_attention_heads,
              encoder_seq_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )
          __snake_case				:			Any            =    decoder_seq_length
          __snake_case				:			List[Any]            =    (
              batch,
              num_decoder_attention_heads,
              decoder_past_length,
              self._config.hidden_size // num_decoder_attention_heads,
          )
          __snake_case				:			Any            =    torch.cat(
              [common_inputs["decoder_attention_mask"], torch.ones(UpperCAmelCase			,  UpperCAmelCase				)]			,  dim=1				)
          __snake_case				:			Any            =    []
          __snake_case      ,				__snake_case				:			Tuple            =    self.num_layers

          for _ in range(UpperCAmelCase				):
            common_inputs["past_key_values"].append(
                (
                    torch.zeros(UpperCAmelCase				),
                    torch.zeros(UpperCAmelCase				),
                    torch.zeros(UpperCAmelCase				),
                    torch.zeros(UpperCAmelCase				),
                )				)
        return common_inputs







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase = -1			,  UpperCAmelCase = -1			,  UpperCAmelCase = False			,  UpperCAmelCase = None			,  )    ->				Mapping[str, Any]:






        '''simple docstring'''






        __snake_case				:			List[Any]            =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)

        if self.use_past:
          if not is_torch_available():
            raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
          else:
            import torch
          __snake_case      ,				__snake_case				:			List[Any]            =    common_inputs["input_ids"].shape
          __snake_case				:			Any            =    seqlen
          __snake_case      ,				__snake_case				:			Dict            =    self.num_layers
          __snake_case      ,				__snake_case				:			int            =    self.num_attention_heads
          __snake_case				:			str            =    (
              batch,
              num_encoder_attention_heads,
              past_key_values_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )
          __snake_case				:			Dict            =    common_inputs["attention_mask"].dtype
          __snake_case				:			Optional[Any]            =    torch.cat(
              [common_inputs["attention_mask"], torch.ones(UpperCAmelCase			,  UpperCAmelCase			,  dtype=UpperCAmelCase				)]			,  dim=1				)
          __snake_case				:			str            =    [
              (torch.zeros(UpperCAmelCase				), torch.zeros(UpperCAmelCase				)) for _ in range(UpperCAmelCase				)
          ]
        return common_inputs







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase = -1			,  UpperCAmelCase = -1			,  UpperCAmelCase = False			,  UpperCAmelCase = None			,  )    ->				Mapping[str, Any]:






        '''simple docstring'''






        __snake_case				:			Tuple            =    compute_effective_axis_dimension(
            UpperCAmelCase			,  fixed_dimension=OnnxConfig.default_fixed_batch			,  num_token_to_add=0				)

        # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
        __snake_case				:			str            =    tokenizer.num_special_tokens_to_add(UpperCAmelCase				)
        __snake_case				:			Any            =    compute_effective_axis_dimension(
            UpperCAmelCase			,  fixed_dimension=OnnxConfig.default_fixed_sequence			,  num_token_to_add=UpperCAmelCase				)

        # Generate dummy inputs according to compute batch and sequence
        __snake_case				:			Tuple            =    [" ".join([tokenizer.unk_token]				) * seq_length] * batch_size
        __snake_case				:			str            =    dict(tokenizer(UpperCAmelCase			,  return_tensors=UpperCAmelCase				)				)
        return common_inputs







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase = -1			,  UpperCAmelCase = -1			,  UpperCAmelCase = False			,  UpperCAmelCase = None			,  )    ->				Mapping[str, Any]:






        '''simple docstring'''






        if self.task in ["default", "seq2seq-lm"]:
          __snake_case				:			Any            =    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
              UpperCAmelCase			,  batch_size=UpperCAmelCase			,  seq_length=UpperCAmelCase			,  is_pair=UpperCAmelCase			,  framework=UpperCAmelCase				)

        elif self.task == "causal-lm":
          __snake_case				:			Union[str, Any]            =    self._generate_dummy_inputs_for_causal_lm(
              UpperCAmelCase			,  batch_size=UpperCAmelCase			,  seq_length=UpperCAmelCase			,  is_pair=UpperCAmelCase			,  framework=UpperCAmelCase				)
        else:
          __snake_case				:			Dict            =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
              UpperCAmelCase			,  batch_size=UpperCAmelCase			,  seq_length=UpperCAmelCase			,  is_pair=UpperCAmelCase			,  framework=UpperCAmelCase				)

        return common_inputs







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)    ->				int:






        '''simple docstring'''






        if self.task in ["default", "seq2seq-lm"]:
          __snake_case				:			str            =    super()._flatten_past_key_values_(UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)
        else:
          __snake_case				:			int            =    super(UpperCAmelCase			,  self				)._flatten_past_key_values_(
              UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase				)







      def   UpperCAmelCase						(	self			,  UpperCAmelCase			,  UpperCAmelCase				)    ->				Any:






        '''simple docstring'''






        if direction not in ["inputs", "outputs"]:
          raise ValueError(F"""direction must either be \"inputs\" or \"outputs\", but {direction} was given"""				)

        __snake_case				:			Tuple            =    "past_key_values" if direction == "inputs" else "present"
        __snake_case      ,				__snake_case				:			Any            =    self.num_layers

        __snake_case				:			int            =    "past_encoder_sequence"
        __snake_case				:			int            =    "past_decoder_sequence" if direction == "inputs" else "past_decoder_sequence + sequence"

        for i in range(UpperCAmelCase				):
          __snake_case				:			Any            =    {0: "batch", 2: decoder_sequence}
          __snake_case				:			List[Any]            =    {0: "batch", 2: decoder_sequence}
          __snake_case				:			Tuple            =    {0: "batch", 2: encoder_sequence}
          __snake_case				:			int            =    {0: "batch", 2: encoder_sequence}

