"""simple docstring"""





import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
       import importlib.resources as pkg_resources
except ImportError:
       # Try backported to PY<37 `importlib_resources`.
       import importlib_resources as pkg_resources

from . import resources


_UpperCamelCase       :						Optional[int]     =   "https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
_UpperCamelCase       :						Union[str, Any]     =   f'''{BASE_REF_URL}/{__file__}'''
_UpperCamelCase       :						Any     =   logging.getLogger(__name__)





def     a_					(				_lowerCAmelCase  :			str					):





       '''simple docstring'''

       lowercase__      :		Optional[Any]          =     pkg_resources.read_text(_lowerCAmelCase       ,			_lowerCAmelCase					)
       return yaml.safe_load(_lowerCAmelCase					), f"""{BASE_REF_URL}/resources/{resource}"""


_UpperCamelCase   ,				_UpperCamelCase       :						int     =   load_yaml_resource("readme_structure.yaml")

_UpperCamelCase       :						Optional[Any]     =   [
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCamelCase       :						List[Any]     =   Tuple[dict, List[str], List[str]]


class 					UpperCAmelCase_     :


      def __init__(       self		,      a		,      a		,      a = None		,      a = False							)	-> str:
             lowercase__      :		List[str]          =     name
             lowercase__      :		int          =     level
             lowercase__      :		Dict          =     lines
             lowercase__      :		Optional[int]          =     ''
             lowercase__      :		List[str]          =     True
             lowercase__      :		Union[str, Any]          =     {}
             lowercase__      :		Any          =     []
             lowercase__      :		Optional[int]          =     []
             if self.lines is not None:
                    self.parse(suppress_parsing_errors=a							)


      def 		_UpperCAmelCase	(       self		,      a = False							)	-> Any:
             lowercase__      :		List[str]          =     ''
             lowercase__      :		int          =     []
             lowercase__      :		Any          =     False
             for line in self.lines:
                    if line.strip(' \n'							) == "":
                           continue
                    elif line.strip(' \n'							)[:3] == "```":
                           lowercase__      :		Optional[Any]          =     not code_start
                    elif line.split()[0] == self.level + "#" and not code_start:
                           if current_sub_level != "":
                                  lowercase__      :		Tuple          =     Section(a		,      self.level + '#'		,      a							)
                                  lowercase__      :		str          =     []
                           else:
                                  if current_lines != []:
                                         self.text += "".join(a							).strip()
                                         if self.text != "" and self.text not in FILLER_TEXT:
                                                lowercase__      :		List[Any]          =     False
                                         lowercase__      :		List[str]          =     []

                           lowercase__      :		Dict          =     ' '.join(line.split()[1:]							).strip(' \n'							)
                    else:
                           current_lines.append(a							)
             else:
                    if current_sub_level != "":
                           if current_sub_level in self.content:
                                  self.parsing_error_list.append(
                                      f"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""							)
                           lowercase__      :		Tuple          =     Section(a		,      self.level + '#'		,      a							)
                    else:
                           if current_lines != []:
                                  self.text += "".join(a							).strip()
                                  if self.text != "" and self.text not in FILLER_TEXT:
                                         lowercase__      :		Dict          =     False

             if self.level == "" and not suppress_parsing_errors:
                    if self.parsing_error_list != [] or self.parsing_warning_list != []:
                           lowercase__      :		Optional[Any]          =     '\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list							)
                           lowercase__      :		int          =     f"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
                           raise ValueError(a							)


      def 		_UpperCAmelCase	(       self		,      a							)	-> ReadmeValidatorOutput:
             lowercase__      :		Tuple          =     []
             lowercase__      :		int          =     []
             if structure["allow_empty"] is False:
                    # If content is expected
                    if self.is_empty_text and self.content == {}:
                           # If no content is found, mention it in the error_list
                           error_list.append(f"""Expected some content in section `{self.name}` but it is empty."""							)

             if structure["allow_empty_text"] is False:
                    # If some text is expected
                    if self.is_empty_text:
                           # If no text is found, mention it in the error_list
                           error_list.append(
                               f"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""							)
        # Subsections Validation
             if structure["subsections"] is not None:
                    # If subsections are expected
                    if self.content == {}:
                           # If no subsections are present
                           lowercase__      :		str          =     [subsection['name'] for subsection in structure['subsections']]
                           # Mention the expected values in the error_list
                           error_list.append(
                               f"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]							)}. Found 'None'."""							)
                    else:
                           # If some subsections are present
                           lowercase__      :		Union[str, Any]          =     [subsection['name'] for subsection in structure['subsections']]
                           lowercase__      :		Any          =     False
                           for idx, name in enumerate(a							):
                                  if name not in self.content:
                                         # If the expected subsection is not present
                                         error_list.append(f"""Section `{self.name}` is missing subsection: `{name}`."""							)
                                         lowercase__      :		str          =     True
                                  else:
                                         # If the subsection is present, validate subsection, return the result
                                         # and concat the errors from subsection to section error_list

                                         # Skip sublevel validation if current level is `###`
                                         if self.level == "###":
                                                continue
                                         else:
                                                lowercase__   ,				lowercase__   ,				lowercase__      :		Optional[Any]          =     self.content[name].validate(
                                                    structure['subsections'][idx]							)
                                         error_list += subsec_error_list
                                         warning_list += subsec_warning_list

                           if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                                  for name in self.content:
                                         if name not in structure_names:
                                                # If an extra subsection is present
                                                warning_list.append(
                                                    f"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""							)
             if error_list:
                    # If there are errors, do not return the dictionary as it is invalid
                    return {}, error_list, warning_list
             else:
                    return self.to_dict(), error_list, warning_list


      def 		_UpperCAmelCase	(       self							)	-> dict:
             return {
                 "name": self.name,
                 "text": self.text,
                 "is_empty_text": self.is_empty_text,
                 "subsections": [value.to_dict() for value in self.content.values()],
             }





class 					UpperCAmelCase_     (   _a):  # Level 0


      def __init__(       self		,      a		,      a		,      a = None		,      a = False							)	-> Any:
             super().__init__(name=a		,      level=''							)  # Not using lines here as we need to use a child class parse
             lowercase__      :		List[Any]          =     structure
             lowercase__      :		Dict          =     -2
             lowercase__      :		Union[str, Any]          =     0
             lowercase__      :		Dict          =     lines
             if self.lines is not None:
                    self.parse(suppress_parsing_errors=a							)


      def 		_UpperCAmelCase	(       self							)	-> Tuple:
             if self.structure is None:
                    lowercase__   ,				lowercase__   ,				lowercase__      :		Any          =     self._validate(a							)
             else:
                    lowercase__   ,				lowercase__   ,				lowercase__      :		Dict          =     self._validate(self.structure							)
             if error_list != [] or warning_list != []:
                    lowercase__      :		Optional[Any]          =     '\n'.join(['-\t' + x for x in error_list + warning_list]							)
                    lowercase__      :		Union[str, Any]          =     f"""The following issues were found for the README at `{self.name}`:\n""" + errors
                    raise ValueError(a							)


      @classmethod
      def 		_UpperCAmelCase	(       cls		,      a		,      a = None		,      a = False							)	-> Tuple:
             with open(a		,      encoding='utf-8'							) as f:
                    lowercase__      :		List[Any]          =     f.readlines()
             return cls(a		,      a		,      a		,      suppress_parsing_errors=a							)


      @classmethod
      def 		_UpperCAmelCase	(       cls		,      a		,      a = None		,      a = "root"		,      a = False							)	-> int:
             lowercase__      :		Optional[Any]          =     string.split('\n'							)
             return cls(a		,      a		,      a		,      suppress_parsing_errors=a							)


      def 		_UpperCAmelCase	(       self		,      a = False							)	-> List[Any]:
             # Skip Tags
             lowercase__      :		Tuple          =     0

             for line in self.lines:
                    self.yaml_tags_line_count += 1
                    if line.strip(' \n'							) == "---":
                           self.tag_count += 1
                           if self.tag_count == 2:
                                  break
                    line_count += 1
             if self.tag_count == 2:
                    lowercase__      :		Optional[int]          =     self.lines[line_count + 1 :]  # Get the last + 1 th item.
             else:
                    lowercase__      :		List[str]          =     self.lines[self.tag_count :]
             super().parse(suppress_parsing_errors=a							)


      def __str__(       self							)	-> Any:
             return str(self.to_dict()							)


      def 		_UpperCAmelCase	(       self		,      a							)	-> str:
             lowercase__      :		Any          =     []
             lowercase__      :		Optional[int]          =     []
             if self.yaml_tags_line_count == 0:
                    warning_list.append('Empty YAML markers are present in the README.'							)
             elif self.tag_count == 0:
                    warning_list.append('No YAML markers are present in the README.'							)
             elif self.tag_count == 1:
                    warning_list.append('Only the start of YAML tags present in the README.'							)
             # Check how many first level sections are present.
             lowercase__      :		int          =     len(self.content.keys()							)
             if num_first_level_keys > 1:
                    # If more than one, add to the error list, continue
                    error_list.append(
                        f"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()							)]							)}. Only one heading is expected. Skipping further validation for this README."""							)
             elif num_first_level_keys < 1:
                    # If less than one, append error.
                    error_list.append(
                        'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'							)

             else:
                    # If one exactly
                    lowercase__      :		Dict          =     list(self.content.keys()							)[0]  # Get the key
                    if start_key.startswith('Dataset Card for'							):  # Check correct start
                           # If the starting is correct, validate all the sections
                           lowercase__   ,				lowercase__   ,				lowercase__      :		Any          =     self.content[start_key].validate(
                               readme_structure['subsections'][0]							)
                           error_list += sec_error_list
                           warning_list += sec_warning_list
                    else:
                           # If not found, append error
                           error_list.append(
                               'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'							)
             if error_list:
                    # If there are errors, do not return the dictionary as it is invalid
                    return {}, error_list, warning_list
             else:
                    return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
       from argparse import ArgumentParser

       _UpperCamelCase       :						Tuple     =   ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
       ap.add_argument("readme_filepath")
       _UpperCamelCase       :						str     =   ap.parse_args()
       _UpperCamelCase       :						Optional[int]     =   Path(args.readme_filepath)
       _UpperCamelCase       :						Dict     =   ReadMe.from_readme(readme_filepath)


"""simple docstring"""





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode





def     a_					(				_lowerCAmelCase  :			Tuple       ,			**_lowerCAmelCase  :			Any					):





       '''simple docstring'''

       return RunBeamCommand(
           args.dataset       ,			args.name       ,			args.cache_dir       ,			args.beam_pipeline_options       ,			args.data_dir       ,			args.all_configs       ,			args.save_info or args.save_infos       ,			args.ignore_verifications       ,			args.force_redownload       ,			**_lowerCAmelCase       ,			)






class 					UpperCAmelCase_     (   _a):


      @staticmethod
      def 		_UpperCAmelCase	(       a							)	-> Optional[int]:
             lowercase__      :		Optional[int]          =     parser.add_parser('run_beam'		,      help='Run a Beam dataset processing pipeline'							)
             run_beam_parser.add_argument('dataset'		,      type=a		,      help='Name of the dataset to download'							)
             run_beam_parser.add_argument('--name'		,      type=a		,      default=a		,      help='Dataset config name'							)
             run_beam_parser.add_argument(
                 '--cache_dir'		,      type=a		,      default=a		,      help='Cache directory where the datasets are stored'		,      )
             run_beam_parser.add_argument(
                 '--beam_pipeline_options'		,      type=a		,      default=''		,      help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'		,      )
             run_beam_parser.add_argument(
                 '--data_dir'		,      type=a		,      default=a		,      help='Can be used to specify a manual directory to get the files from'		,      )
             run_beam_parser.add_argument('--all_configs'		,      action='store_true'		,      help='Test all dataset configurations'							)
             run_beam_parser.add_argument('--save_info'		,      action='store_true'		,      help='Save the dataset infos file'							)
             run_beam_parser.add_argument(
                 '--ignore_verifications'		,      action='store_true'		,      help='Run the test without checksums and splits checks'							)
             run_beam_parser.add_argument('--force_redownload'		,      action='store_true'		,      help='Force dataset redownload'							)
             # aliases
             run_beam_parser.add_argument('--save_infos'		,      action='store_true'		,      help='alias for save_info'							)
             run_beam_parser.set_defaults(func=a							)


      def __init__(       self		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      a		,      **a		,      )	-> Any:
             lowercase__      :		List[Any]          =     dataset
             lowercase__      :		List[Any]          =     name
             lowercase__      :		List[Any]          =     cache_dir
             lowercase__      :		Any          =     beam_pipeline_options
             lowercase__      :		List[str]          =     data_dir
             lowercase__      :		int          =     all_configs
             lowercase__      :		Optional[Any]          =     save_infos
             lowercase__      :		List[Any]          =     ignore_verifications
             lowercase__      :		int          =     force_redownload
             lowercase__      :		List[Any]          =     config_kwargs


      def 		_UpperCAmelCase	(       self							)	-> int:
             import apache_beam as beam

             if self._name is not None and self._all_configs:
                    print('Both parameters `name` and `all_configs` can\'t be used at once.'							)
                    exit(1							)
             lowercase__   ,				lowercase__      :		Optional[int]          =     self._dataset, self._name
             lowercase__      :		List[str]          =     dataset_module_factory(a							)
             lowercase__      :		Dict          =     import_main_class(dataset_module.module_path							)
             lowercase__      :		List[DatasetBuilder]          =     []
             if self._beam_pipeline_options:
                    lowercase__      :		Any          =     beam.options.pipeline_options.PipelineOptions(
                        flags=[f"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(','							) if opt]							)
             else:
                    lowercase__      :		Optional[int]          =     None
             if self._all_configs and len(builder_cls.BUILDER_CONFIGS							) > 0:
                    for builder_config in builder_cls.BUILDER_CONFIGS:
                           builders.append(
                               builder_cls(
                                   config_name=builder_config.name		,      data_dir=self._data_dir		,      hash=dataset_module.hash		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      )							)
             else:
                    builders.append(
                        builder_cls(
                            config_name=a		,      data_dir=self._data_dir		,      beam_options=a		,      cache_dir=self._cache_dir		,      base_path=dataset_module.builder_kwargs.get('base_path'							)		,      **self._config_kwargs		,      )							)

             for builder in builders:
                    builder.download_and_prepare(
                        download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                        if not self._force_redownload
                        else DownloadMode.FORCE_REDOWNLOAD		,      download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH							)		,      verification_mode=VerificationMode.NO_CHECKS
                        if self._ignore_verifications
                        else VerificationMode.ALL_CHECKS		,      try_from_hf_gcs=a		,      )
                    if self._save_infos:
                           builder._save_infos()

             print('Apache beam run successful.'							)

             # If save_infos=True, the dataset infos file is created next to the loaded module file.
             # Let's move it to the original directory of the dataset script, to allow the user to
             # upload them on S3 at the same time afterwards.
             if self._save_infos:
                    lowercase__      :		Dict          =     os.path.join(builder_cls.get_imported_module_dir()		,      config.DATASETDICT_INFOS_FILENAME							)

                    lowercase__      :		Union[str, Any]          =     Path(a							).name + '.py'

                    lowercase__      :		List[Any]          =     os.path.join(a		,      a							)
                    if os.path.isfile(a							):
                           lowercase__      :		Any          =     os.path.dirname(a							)
                    elif os.path.isfile(a							):
                           lowercase__      :		Optional[int]          =     path
                    else:  # in case of a remote dataset
                           print(f"""Dataset Infos file saved at {dataset_infos_path}"""							)
                           exit(1							)

                    # Move datasetinfo back to the user
                    lowercase__      :		List[Any]          =     os.path.join(a		,      config.DATASETDICT_INFOS_FILENAME							)
                    copyfile(a		,      a							)
                    print(f"""Dataset Infos file saved at {user_dataset_infos_path}"""							)

