

import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
 import importlib.resources as pkg_resources
except ImportError:
 # Try backported to PY<37 `importlib_resources`.
 import importlib_resources as pkg_resources

from . import resources


_UpperCamelCase			      =		"https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
_UpperCamelCase			      =		F"{BASE_REF_URL}/{__file__}"
_UpperCamelCase			      =		logging.getLogger(__name__)







def 		_lowercase			(					lowercase__						):
      __lowerCAmelCase :						Tuple = pkg_resources.read_text(lowercase__       ,     lowercase__						)
      return yaml.safe_load(lowercase__						), f"""{BASE_REF_URL}/resources/{resource}"""


_UpperCamelCase      ,      _UpperCamelCase			      =		load_yaml_resource("readme_structure.yaml")

_UpperCamelCase			      =		[
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCamelCase			      =		Tuple[dict, List[str], List[str]]







class   __lowercase				:
 def __init__(							self				,							A_				,							A_				,							A_ = None				,							A_ = False ) ->List[str]:



       '''simple docstring'''





       __lowerCAmelCase :						Dict = name
       __lowerCAmelCase :						str = level
       __lowerCAmelCase :						str = lines
       __lowerCAmelCase :						List[Any] = ''''''
       __lowerCAmelCase :						Optional[Any] = True
       __lowerCAmelCase :						Optional[Any] = {}
       __lowerCAmelCase :						int = []
       __lowerCAmelCase :						Any = []
       if self.lines is not None:
             self.parse(suppress_parsing_errors=A_ )
 def      UpperCamelCase__  (							self				,							A_ = False ) ->str:



       '''simple docstring'''





       __lowerCAmelCase :						List[Any] = ''''''
       __lowerCAmelCase :						str = []
       __lowerCAmelCase :						Tuple = False
       for line in self.lines:
             if line.strip(''' \n''' ) == "":
                   continue
             elif line.strip(''' \n''' )[:3] == "```":
                   __lowerCAmelCase :						Dict = not code_start
             elif line.split()[0] == self.level + "#" and not code_start:
                   if current_sub_level != "":
                         __lowerCAmelCase :						Dict = Section(A_				,							self.level + '''#'''				,							A_ )
                         __lowerCAmelCase :						List[Any] = []
                   else:
                         if current_lines != []:
                               self.text += "".join(A_ ).strip()
                               if self.text != "" and self.text not in FILLER_TEXT:
                                     __lowerCAmelCase :						List[str] = False
                               __lowerCAmelCase :						int = []

                   __lowerCAmelCase :						Optional[Any] = ''' '''.join(line.split()[1:] ).strip(''' \n''' )
             else:
                   current_lines.append(A_ )
       else:
             if current_sub_level != "":
                   if current_sub_level in self.content:
                         self.parsing_error_list.append(
                             f"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.""" )
                   __lowerCAmelCase :						Dict = Section(A_				,							self.level + '''#'''				,							A_ )
             else:
                   if current_lines != []:
                         self.text += "".join(A_ ).strip()
                         if self.text != "" and self.text not in FILLER_TEXT:
                               __lowerCAmelCase :						Dict = False

       if self.level == "" and not suppress_parsing_errors:
             if self.parsing_error_list != [] or self.parsing_warning_list != []:
                   __lowerCAmelCase :						List[str] = '''\n'''.join('''-\t''' + x for x in self.parsing_error_list + self.parsing_warning_list )
                   __lowerCAmelCase :						List[Any] = f"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
                   raise ValueError(A_ )
 def      UpperCamelCase__  (							self				,							A_ ) ->ReadmeValidatorOutput:



       '''simple docstring'''





       __lowerCAmelCase :						Dict = []
       __lowerCAmelCase :						Dict = []
       if structure["allow_empty"] is False:
             # If content is expected
             if self.is_empty_text and self.content == {}:
                   # If no content is found, mention it in the error_list
                   error_list.append(f"""Expected some content in section `{self.name}` but it is empty.""" )

       if structure["allow_empty_text"] is False:
             # If some text is expected
             if self.is_empty_text:
                   # If no text is found, mention it in the error_list
                   error_list.append(
                       f"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).""" )
        # Subsections Validation
       if structure["subsections"] is not None:
             # If subsections are expected
             if self.content == {}:
                   # If no subsections are present
                   __lowerCAmelCase :						Optional[Any] = [subsection['''name'''] for subsection in structure['''subsections''']]
                   # Mention the expected values in the error_list
                   error_list.append(
                       f"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values] )}. Found 'None'.""" )
             else:
                   # If some subsections are present
                   __lowerCAmelCase :						Optional[Any] = [subsection['''name'''] for subsection in structure['''subsections''']]
                   __lowerCAmelCase :						Union[str, Any] = False
                   for idx, name in enumerate(A_ ):
                         if name not in self.content:
                               # If the expected subsection is not present
                               error_list.append(f"""Section `{self.name}` is missing subsection: `{name}`.""" )
                               __lowerCAmelCase :						Tuple = True
                         else:
                               # If the subsection is present, validate subsection, return the result
                               # and concat the errors from subsection to section error_list

                               # Skip sublevel validation if current level is `###`
                               if self.level == "###":
                                     continue
                               else:
                                     __lowerCAmelCase,							__lowerCAmelCase,							__lowerCAmelCase :						List[Any] = self.content[name].validate(
                                         structure['''subsections'''][idx] )
                               error_list += subsec_error_list
                               warning_list += subsec_warning_list

                   if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                         for name in self.content:
                               if name not in structure_names:
                                     # If an extra subsection is present
                                     warning_list.append(
                                         f"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.""" )
       if error_list:
             # If there are errors, do not return the dictionary as it is invalid
             return {}, error_list, warning_list
       else:
             return self.to_dict(), error_list, warning_list





 def      UpperCamelCase__  (							self ) ->dict:



       '''simple docstring'''





       return {
           "name": self.name,
           "text": self.text,
           "is_empty_text": self.is_empty_text,
           "subsections": [value.to_dict() for value in self.content.values()],
       }







class   __lowercase				(_UpperCAmelCase			):  # Level 0
 def __init__(							self				,							A_				,							A_				,							A_ = None				,							A_ = False ) ->List[str]:



       '''simple docstring'''





       super().__init__(name=A_				,							level='''''' )  # Not using lines here as we need to use a child class parse
       __lowerCAmelCase :						List[Any] = structure
       __lowerCAmelCase :						Dict = -2
       __lowerCAmelCase :						List[Any] = 0
       __lowerCAmelCase :						List[str] = lines
       if self.lines is not None:
             self.parse(suppress_parsing_errors=A_ )
 def      UpperCamelCase__  (							self ) ->Any:



       '''simple docstring'''





       if self.structure is None:
             __lowerCAmelCase,							__lowerCAmelCase,							__lowerCAmelCase :						List[str] = self._validate(A_ )
       else:
             __lowerCAmelCase,							__lowerCAmelCase,							__lowerCAmelCase :						Optional[int] = self._validate(self.structure )
       if error_list != [] or warning_list != []:
             __lowerCAmelCase :						int = '''\n'''.join(['''-\t''' + x for x in error_list + warning_list] )
             __lowerCAmelCase :						Optional[Any] = f"""The following issues were found for the README at `{self.name}`:\n""" + errors
             raise ValueError(A_ )
 @classmethod
 def      UpperCamelCase__  (							cls				,							A_				,							A_ = None				,							A_ = False ) ->Any:



       '''simple docstring'''





       with open(A_				,							encoding='''utf-8''' ) as f:
             __lowerCAmelCase :						int = f.readlines()
       return cls(A_				,							A_				,							A_				,							suppress_parsing_errors=A_ )
 @classmethod
 def      UpperCamelCase__  (							cls				,							A_				,							A_ = None				,							A_ = "root"				,							A_ = False ) ->Optional[int]:



       '''simple docstring'''





       __lowerCAmelCase :						Optional[Any] = string.split('''\n''' )
       return cls(A_				,							A_				,							A_				,							suppress_parsing_errors=A_ )
 def      UpperCamelCase__  (							self				,							A_ = False ) ->Dict:



       '''simple docstring'''





       __lowerCAmelCase :						str = 0

       for line in self.lines:
             self.yaml_tags_line_count += 1
             if line.strip(''' \n''' ) == "---":
                   self.tag_count += 1
                   if self.tag_count == 2:
                         break
             line_count += 1
       if self.tag_count == 2:
             __lowerCAmelCase :						Union[str, Any] = self.lines[line_count + 1 :]  # Get the last + 1 th item.
       else:
             __lowerCAmelCase :						Dict = self.lines[self.tag_count :]
       super().parse(suppress_parsing_errors=A_ )
 def __str__(							self ) ->int:



       '''simple docstring'''





       return str(self.to_dict() )





 def      UpperCamelCase__  (							self				,							A_ ) ->int:



       '''simple docstring'''





       __lowerCAmelCase :						List[str] = []
       __lowerCAmelCase :						str = []
       if self.yaml_tags_line_count == 0:
             warning_list.append('''Empty YAML markers are present in the README.''' )
       elif self.tag_count == 0:
             warning_list.append('''No YAML markers are present in the README.''' )
       elif self.tag_count == 1:
             warning_list.append('''Only the start of YAML tags present in the README.''' )
       # Check how many first level sections are present.
       __lowerCAmelCase :						Union[str, Any] = len(self.content.keys() )
       if num_first_level_keys > 1:
             # If more than one, add to the error list, continue
             error_list.append(
                 f"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys() )] )}. Only one heading is expected. Skipping further validation for this README.""" )
       elif num_first_level_keys < 1:
             # If less than one, append error.
             error_list.append(
                 '''The README has no first-level headings. One heading is expected. Skipping further validation for this README.''' )

       else:
             # If one exactly
             __lowerCAmelCase :						Any = list(self.content.keys() )[0]  # Get the key
             if start_key.startswith('''Dataset Card for''' ):  # Check correct start
                   # If the starting is correct, validate all the sections
                   __lowerCAmelCase,							__lowerCAmelCase,							__lowerCAmelCase :						Optional[int] = self.content[start_key].validate(
                       readme_structure['''subsections'''][0] )
                   error_list += sec_error_list
                   warning_list += sec_warning_list
             else:
                   # If not found, append error
                   error_list.append(
                       '''No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.''' )
       if error_list:
             # If there are errors, do not return the dictionary as it is invalid
             return {}, error_list, warning_list
       else:
             return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
 from argparse import ArgumentParser

 _UpperCamelCase			      =		ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
 ap.add_argument("readme_filepath")
 _UpperCamelCase			      =		ap.parse_args()
 _UpperCamelCase			      =		Path(args.readme_filepath)
 _UpperCamelCase			      =		ReadMe.from_readme(readme_filepath)





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import BlipImageProcessor







class   __lowercase				(unittest.TestCase			):
 def __init__(							self				,							A_				,							A_=7				,							A_=3				,							A_=18				,							A_=30				,							A_=400				,							A_=True				,							A_=None				,							A_=True				,							A_=False				,							A_=[0.48_145_466, 0.4_578_275, 0.40_821_073]				,							A_=[0.26_862_954, 0.26_130_258, 0.27_577_711]				,							A_=True				,							) ->Dict:



       '''simple docstring'''





       __lowerCAmelCase :						Union[str, Any] = size if size is not None else {'''height''': 20, '''width''': 20}
       __lowerCAmelCase :						List[str] = parent
       __lowerCAmelCase :						List[str] = batch_size
       __lowerCAmelCase :						Optional[Any] = num_channels
       __lowerCAmelCase :						List[Any] = image_size
       __lowerCAmelCase :						Optional[int] = min_resolution
       __lowerCAmelCase :						Optional[int] = max_resolution
       __lowerCAmelCase :						List[Any] = do_resize
       __lowerCAmelCase :						Dict = size
       __lowerCAmelCase :						Dict = do_normalize
       __lowerCAmelCase :						Any = image_mean
       __lowerCAmelCase :						Optional[int] = image_std
       __lowerCAmelCase :						Any = do_pad
       __lowerCAmelCase :						Any = do_convert_rgb
 def      UpperCamelCase__  (							self ) ->List[str]:



       '''simple docstring'''





       return {
           "do_resize": self.do_resize,
           "size": self.size,
           "do_normalize": self.do_normalize,
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_convert_rgb": self.do_convert_rgb,
           "do_pad": self.do_pad,
       }





 def      UpperCamelCase__  (							self				,							A_=False				,							A_=False				,							A_=False ) ->Dict:



       '''simple docstring'''





       assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

       if equal_resolution:
             __lowerCAmelCase :						int = []
             for i in range(self.batch_size ):
                   image_inputs.append(
                       np.random.randint(
                           255				,							size=(self.num_channels, self.max_resolution, self.max_resolution)				,							dtype=np.uinta ) )
       else:
             __lowerCAmelCase :						List[Any] = []
             for i in range(self.batch_size ):
                   __lowerCAmelCase,							__lowerCAmelCase :						Any = np.random.choice(np.arange(self.min_resolution				,							self.max_resolution )				,							2 )
                   image_inputs.append(np.random.randint(255				,							size=(self.num_channels, width, height)				,							dtype=np.uinta ) )

       if not numpify and not torchify:
             # PIL expects the channel dimension as last dimension
             __lowerCAmelCase :						List[str] = [Image.fromarray(np.moveaxis(A_				,							0				,							-1 ) ) for x in image_inputs]

       if torchify:
             __lowerCAmelCase :						int = [torch.from_numpy(A_ ) for x in image_inputs]

       return image_inputs







@require_torch
@require_vision
class   __lowercase				(_UpperCAmelCase		,       unittest.TestCase			):
 _UpperCamelCase        =  BlipImageProcessor if is_vision_available() else None
 def      UpperCamelCase__  (							self ) ->Tuple:



       '''simple docstring'''





       __lowerCAmelCase :						Union[str, Any] = BlipImageProcessingTester(self )
 @property
 def      UpperCamelCase__  (							self ) ->int:



       '''simple docstring'''





       return self.image_processor_tester.prepare_image_processor_dict()
 def      UpperCamelCase__  (							self ) ->int:



       '''simple docstring'''





       __lowerCAmelCase :						str = self.image_processing_class(**self.image_processor_dict )
       self.assertTrue(hasattr(A_				,							'''do_resize''' ) )
       self.assertTrue(hasattr(A_				,							'''size''' ) )
       self.assertTrue(hasattr(A_				,							'''do_normalize''' ) )
       self.assertTrue(hasattr(A_				,							'''image_mean''' ) )
       self.assertTrue(hasattr(A_				,							'''image_std''' ) )
       self.assertTrue(hasattr(A_				,							'''do_convert_rgb''' ) )
 def      UpperCamelCase__  (							self ) ->str:



       '''simple docstring'''





       pass
 def      UpperCamelCase__  (							self ) ->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Dict = self.image_processing_class(**self.image_processor_dict )
       # create random PIL images
       __lowerCAmelCase :						List[str] = self.image_processor_tester.prepare_inputs(equal_resolution=A_ )
       for image in image_inputs:
             self.assertIsInstance(A_				,							Image.Image )

       # Test not batched input
       __lowerCAmelCase :						Optional[Any] = image_processor(image_inputs[0]				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)

       # Test batched
       __lowerCAmelCase :						str = image_processor(A_				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)
 def      UpperCamelCase__  (							self ) ->Union[str, Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Optional[Any] = self.image_processing_class(**self.image_processor_dict )
       # create random numpy tensors
       __lowerCAmelCase :						Union[str, Any] = self.image_processor_tester.prepare_inputs(equal_resolution=A_				,							numpify=A_ )
       for image in image_inputs:
             self.assertIsInstance(A_				,							np.ndarray )

       # Test not batched input
       __lowerCAmelCase :						Dict = image_processor(image_inputs[0]				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)

       # Test batched
       __lowerCAmelCase :						List[Any] = image_processor(A_				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)





 def      UpperCamelCase__  (							self ) ->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase :						List[Any] = self.image_processing_class(**self.image_processor_dict )
       # create random PyTorch tensors
       __lowerCAmelCase :						str = self.image_processor_tester.prepare_inputs(equal_resolution=A_				,							torchify=A_ )
       for image in image_inputs:
             self.assertIsInstance(A_				,							torch.Tensor )

       # Test not batched input
       __lowerCAmelCase :						List[Any] = image_processor(image_inputs[0]				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)

       # Test batched
       __lowerCAmelCase :						Tuple = image_processor(A_				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)







@require_torch
@require_vision
class   __lowercase				(_UpperCAmelCase		,       unittest.TestCase			):
 _UpperCamelCase        =  BlipImageProcessor if is_vision_available() else None
 def      UpperCamelCase__  (							self ) ->Optional[Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Any = BlipImageProcessingTester(self				,							num_channels=4 )
       __lowerCAmelCase :						str = 3
 @property
 def      UpperCamelCase__  (							self ) ->List[str]:



       '''simple docstring'''





       return self.image_processor_tester.prepare_image_processor_dict()
 def      UpperCamelCase__  (							self ) ->Tuple:



       '''simple docstring'''





       __lowerCAmelCase :						int = self.image_processing_class(**self.image_processor_dict )
       self.assertTrue(hasattr(A_				,							'''do_resize''' ) )
       self.assertTrue(hasattr(A_				,							'''size''' ) )
       self.assertTrue(hasattr(A_				,							'''do_normalize''' ) )
       self.assertTrue(hasattr(A_				,							'''image_mean''' ) )
       self.assertTrue(hasattr(A_				,							'''image_std''' ) )
       self.assertTrue(hasattr(A_				,							'''do_convert_rgb''' ) )
 def      UpperCamelCase__  (							self ) ->Union[str, Any]:



       '''simple docstring'''





       pass





 def      UpperCamelCase__  (							self ) ->Any:



       '''simple docstring'''





       __lowerCAmelCase :						Tuple = self.image_processing_class(**self.image_processor_dict )
       # create random PIL images
       __lowerCAmelCase :						Optional[int] = self.image_processor_tester.prepare_inputs(equal_resolution=A_ )
       for image in image_inputs:
             self.assertIsInstance(A_				,							Image.Image )

       # Test not batched input
       __lowerCAmelCase :						Tuple = image_processor(image_inputs[0]				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               1,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)

       # Test batched
       __lowerCAmelCase :						List[str] = image_processor(A_				,							return_tensors='''pt''' ).pixel_values
       self.assertEqual(
           encoded_images.shape				,							(
               self.image_processor_tester.batch_size,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.size['''height'''],
               self.image_processor_tester.size['''width'''],
           )				,							)


