"""simple docstring"""





import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


_UpperCamelCase       :						List[Any]     =   get_logger(__name__)

# Datasets
_UpperCamelCase       :						int     =   "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
_UpperCamelCase       :						Tuple     =   "https://cdn-datasets.huggingface.co/datasets/datasets"
_UpperCamelCase       :						int     =   "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"

# Metrics
_UpperCamelCase       :						str     =   "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"
_UpperCamelCase       :						Optional[Any]     =   "https://cdn-datasets.huggingface.co/datasets/metric"
_UpperCamelCase       :						Optional[Any]     =   "https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"

# Hub
_UpperCamelCase       :						List[Any]     =   os.environ.get("HF_ENDPOINT", "https://huggingface.co")
_UpperCamelCase       :						int     =   HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
_UpperCamelCase       :						Tuple     =   "main"

_UpperCamelCase       :						Optional[int]     =   version.parse(platform.python_version())

if PY_VERSION < version.parse("3.8"):
							import importlib_metadata
else:
							import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
_UpperCamelCase       :						List[str]     =   {"1", "ON", "YES", "TRUE"}
_UpperCamelCase       :						Dict     =   ENV_VARS_TRUE_VALUES.union({"AUTO"})


# Imports
_UpperCamelCase       :						Dict     =   version.parse(importlib_metadata.version("dill"))
_UpperCamelCase       :						str     =   version.parse(importlib_metadata.version("pandas"))
_UpperCamelCase       :						int     =   version.parse(importlib_metadata.version("pyarrow"))

_UpperCamelCase       :						int     =   os.environ.get("USE_TF", "AUTO").upper()
_UpperCamelCase       :						List[str]     =   os.environ.get("USE_TORCH", "AUTO").upper()
_UpperCamelCase       :						Union[str, Any]     =   os.environ.get("USE_JAX", "AUTO").upper()

_UpperCamelCase       :						Union[str, Any]     =   "N/A"
_UpperCamelCase       :						List[str]     =   False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
							_UpperCamelCase       :						Any     =   importlib.util.find_spec("torch") is not None
							if TORCH_AVAILABLE:
														try:
																					_UpperCamelCase       :						Tuple     =   version.parse(importlib_metadata.version("torch"))
																					logger.info(f'''PyTorch version {TORCH_VERSION} available.''')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info("Disabling PyTorch because USE_TF is set")

_UpperCamelCase       :						str     =   "N/A"
_UpperCamelCase       :						Any     =   False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
							_UpperCamelCase       :						Union[str, Any]     =   importlib.util.find_spec("tensorflow") is not None
							if TF_AVAILABLE:
														# For the metadata, we have to look for both tensorflow and tensorflow-cpu
														for package in [
														    "tensorflow",
														    "tensorflow-cpu",
														    "tensorflow-gpu",
														    "tf-nightly",
														    "tf-nightly-cpu",
														    "tf-nightly-gpu",
														    "intel-tensorflow",
														    "tensorflow-rocm",
														    "tensorflow-macos",
														]:
																					try:
																												_UpperCamelCase       :						Optional[Any]     =   version.parse(importlib_metadata.version(package))
																					except importlib_metadata.PackageNotFoundError:
																												continue
																					else:
																												break
														else:
																					_UpperCamelCase       :						str     =   False
							if TF_AVAILABLE:
														if TF_VERSION.major < 2:
																					logger.info(f'''TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.''')
																					_UpperCamelCase       :						str     =   False
														else:
																					logger.info(f'''TensorFlow version {TF_VERSION} available.''')
else:
							logger.info("Disabling Tensorflow because USE_TORCH is set")


_UpperCamelCase       :						int     =   "N/A"
_UpperCamelCase       :						Union[str, Any]     =   False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
							_UpperCamelCase       :						Any     =   importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
							if JAX_AVAILABLE:
														try:
																					_UpperCamelCase       :						Optional[int]     =   version.parse(importlib_metadata.version("jax"))
																					logger.info(f'''JAX version {JAX_VERSION} available.''')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info("Disabling JAX because USE_JAX is set to False")


_UpperCamelCase       :						Optional[int]     =   os.environ.get("USE_BEAM", "AUTO").upper()
_UpperCamelCase       :						Tuple     =   "N/A"
_UpperCamelCase       :						Optional[Any]     =   False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
							try:
														_UpperCamelCase       :						Dict     =   version.parse(importlib_metadata.version("apache_beam"))
														_UpperCamelCase       :						Dict     =   True
														logger.info(f'''Apache Beam version {BEAM_VERSION} available.''')
							except importlib_metadata.PackageNotFoundError:
														pass
else:
							logger.info("Disabling Apache Beam because USE_BEAM is set to False")


# Optional tools for data loading
_UpperCamelCase       :						Any     =   importlib.util.find_spec("sqlalchemy") is not None

# Optional tools for feature decoding
_UpperCamelCase       :						Tuple     =   importlib.util.find_spec("PIL") is not None
_UpperCamelCase       :						Optional[int]     =   importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.0.31")
_UpperCamelCase       :						int     =   importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.1.0")

# Optional compression tools
_UpperCamelCase       :						Any     =   importlib.util.find_spec("rarfile") is not None
_UpperCamelCase       :						Optional[int]     =   importlib.util.find_spec("zstandard") is not None
_UpperCamelCase       :						List[Any]     =   importlib.util.find_spec("lz4") is not None
_UpperCamelCase       :						Tuple     =   importlib.util.find_spec("py7zr") is not None


# Cache location
_UpperCamelCase       :						Tuple     =   "~/.cache"
_UpperCamelCase       :						Optional[Any]     =   os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
_UpperCamelCase       :						List[Any]     =   os.path.join(XDG_CACHE_HOME, "huggingface")
_UpperCamelCase       :						Union[str, Any]     =   os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))

_UpperCamelCase       :						Optional[Any]     =   os.path.join(HF_CACHE_HOME, "datasets")
_UpperCamelCase       :						Any     =   Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))

_UpperCamelCase       :						Any     =   os.path.join(HF_CACHE_HOME, "metrics")
_UpperCamelCase       :						str     =   Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE))

_UpperCamelCase       :						Optional[int]     =   os.path.join(HF_CACHE_HOME, "modules")
_UpperCamelCase       :						List[str]     =   Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))

_UpperCamelCase       :						Tuple     =   "downloads"
_UpperCamelCase       :						List[str]     =   os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
_UpperCamelCase       :						List[Any]     =   Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))

_UpperCamelCase       :						Optional[int]     =   "extracted"
_UpperCamelCase       :						Any     =   os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
_UpperCamelCase       :						Optional[Any]     =   Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
_UpperCamelCase       :						List[Any]     =   (
    os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
_UpperCamelCase       :						List[str]     =   10_00

# Size of the preloaded record batch in `Dataset.__iter__`
_UpperCamelCase       :						List[str]     =   10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
_UpperCamelCase       :						Optional[Any]     =   4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
_UpperCamelCase       :						Optional[int]     =   "500MB"

# Parquet configuration
_UpperCamelCase       :						int     =   1_00
_UpperCamelCase       :						Optional[Any]     =   1_00
_UpperCamelCase       :						List[Any]     =   1_00

# Offline mode
_UpperCamelCase       :						Optional[int]     =   os.environ.get("HF_DATASETS_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES

# In-memory
_UpperCamelCase       :						Tuple     =   0  # Disabled
_UpperCamelCase       :						Dict     =   float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
_UpperCamelCase       :						Dict     =   "dataset.arrow"
_UpperCamelCase       :						Optional[Any]     =   "indices.arrow"
_UpperCamelCase       :						Optional[Any]     =   "state.json"
_UpperCamelCase       :						str     =   "dataset_info.json"
_UpperCamelCase       :						Dict     =   "dataset_infos.json"
_UpperCamelCase       :						Any     =   "LICENSE"
_UpperCamelCase       :						str     =   "metric_info.json"
_UpperCamelCase       :						int     =   "dataset_dict.json"

_UpperCamelCase       :						Optional[int]     =   "datasets_modules"

_UpperCamelCase       :						int     =   2_55

# Streaming
_UpperCamelCase       :						Tuple     =   20
_UpperCamelCase       :						str     =   5

# Datasets without script
_UpperCamelCase       :						Tuple     =   2_00
_UpperCamelCase       :						Optional[Any]     =   10
_UpperCamelCase       :						Tuple     =   2_00

# Progress bars
_UpperCamelCase       :						Union[str, Any]     =   0.0_5  # 20 progress updates per sec


"""simple docstring"""





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode





def     a_					(				_lowerCAmelCase  :			Tuple       ,			**_lowerCAmelCase  :			Any					):





       '''simple docstring'''

       return RunBeamCommand(
           args.dataset       ,			args.name       ,			args.cache_dir       ,			args.beam_pipeline_options       ,			args.data_dir       ,			args.all_configs       ,			args.save_info or args.save_infos       ,			args.ignore_verifications       ,			args.force_redownload       ,			**_lowerCAmelCase       ,			)






class 					UpperCAmelCase_     (   _a):


      @staticmethod
      def 		_UpperCAmelCase	(       a							)	-> Optional[int]:
             lowercase__      :		Optional[int]          =     parser.add_parser('run_beam'		,      help='Run a Beam dataset processing pipeline'							)
             run_beam_parser.add_argument('dataset'		,      type=a		,      help='Name of the dataset to download'							)
             run_beam_parser.add_argument('--name'		,      type=a		,      default=a		,      help='Dataset config name'							)
             run_beam_parser.add_argument(
                 '--cache_dir'		,      type=a		,      default=a		,      help='Cache directory where the datasets are stored'		,      )
             run_beam_parser.add_argument(
                 '--beam_pipeline_options'		,      type=a		,      default=''		,      help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'		,      )
             run_beam_parser.add_argument(
                 '--data_dir'		,      type=a		,      default=a		,      help='Can be used to specify a manual directory to get the files from'		,      )
             run_beam_parser.add_argument('--all_configs'		,      action='store_true'		,      help='Test all dataset configurations'							)
             run_beam_parser.add_argument('--save_info'		,      action='store_true'		,      help='Save the dataset infos file'							)
             run_beam_parser.add_argument(
                 '--ignore_verifications'		,      action='store_true'		,      help='Run the test without checksums and splits checks'							)
             run_beam_parser.add_argument('--force_redownload'		,      action='store_true'		,      help='Force dataset redownload'							)
             # aliases
             run_beam_parser.add_argument('--save_infos'		,      action='store_true'		,      help='alias for save_info'							)
             run_beam_parser.set_defaults(func=a							)


      def __init__(       self		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      **a		,      )	-> Any:
             lowercase__      :		List[Any]          =     dataset
             lowercase__      :		List[Any]          =     name
             lowercase__      :		List[Any]          =     cache_dir
             lowercase__      :		Any          =     beam_pipeline_options
             lowercase__      :		List[str]          =     data_dir
             lowercase__      :		int          =     all_configs
             lowercase__      :		Optional[Any]          =     save_infos
             lowercase__      :		List[Any]          =     ignore_verifications
             lowercase__      :		int          =     force_redownload
             lowercase__      :		List[Any]          =     config_kwargs


      def 		_UpperCAmelCase	(       self							)	-> int:
             import apache_beam as beam

             if self._name is not None and self._all_configs:
                    print('Both parameters `name` and `all_configs` can\'t be used at once.'							)
                    exit(1							)
             lowercase__   ,				lowercase__      :		Optional[int]          =     self._dataset, self._name
             lowercase__      :		List[str]          =     dataset_module_factory(a							)
             lowercase__      :		Dict          =     import_main_class(dataset_module.module_path							)
             lowercase__      :		List[DatasetBuilder]          =     []
             if self._beam_pipeline_options:
                    lowercase__      :		Any          =     beam.options.pipeline_options.PipelineOptions(
                        flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(','							) if opt]							)
             else:
                    lowercase__      :		Optional[int]          =     None
             if self._all_configs and len(builder_cls.BUILDER_CONFIGS							) > 0:
                    for builder_config in builder_cls.BUILDER_CONFIGS:
                           builders.append(
                               builder_cls(
                                   config_name=builder_config.name		,      data_dir=self._data_dir		,      hash=dataset_module.hash		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      )							)
             else:
                    builders.append(
                        builder_cls(
                            config_name=a		,      data_dir=self._data_dir		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      **self._config_kwargs		,      )							)

             for builder in builders:
                    builder.download_and_prepare(
                        download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                        if not self._force_redownload
                        else DownloadMode.FORCE_REDOWNLOAD		,      download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH							)		,      verification_mode=VerificationMode.NO_CHECKS
                        if self._ignore_verifications
                        else VerificationMode.ALL_CHECKS		,      try_from_hf_gcs=a		,      )
                    if self._save_infos:
                           builder._save_infos()

             print('Apache beam run successful.'							)

             # If save_infos=True, the dataset infos file is created next to the loaded module file.
             # Let's move it to the original directory of the dataset script, to allow the user to
             # upload them on S3 at the same time afterwards.
             if self._save_infos:
                    lowercase__      :		Dict          =     os.path.join(builder_cls.get_imported_module_dir()		,      config.DATASETDICT_INFOS_FILENAME							)

                    lowercase__      :		Union[str, Any]          =     Path(a							).name + '.py'

                    lowercase__      :		List[Any]          =     os.path.join(a		,      a							)
                    if os.path.isfile(a							):
                           lowercase__      :		Any          =     os.path.dirname(a							)
                    elif os.path.isfile(a							):
                           lowercase__      :		Optional[int]          =     path
                    else:  # in case of a remote dataset
                           print(f"""Dataset Infos file saved at {dataset_infos_path}"""							)
                           exit(1							)

                    # Move datasetinfo back to the user
                    lowercase__      :		List[Any]          =     os.path.join(a		,      config.DATASETDICT_INFOS_FILENAME							)
                    copyfile(a		,      a							)
                    print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""							)

