





"""simple docstring"""




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase						:       Union[str, Any]        =					logging.get_logger(__name__)

UpperCAmelCase						:       List[Any]        =					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase						:       Optional[int]        =					{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

UpperCAmelCase						:       Tuple        =					{
    '''openai-gpt''': 5_12,
}







def 			_SCREAMING_SNAKE_CASE    (       a  )				->				Tuple:
					__A     :						Optional[int]          =    text.strip()
					if not text:
										return []
					__A     :						Any          =    text.split()
					return tokens





class 						_A( snake_case__       ):


				"""simple docstring"""


				def __init__(      self					, _A=True					, _A=None					, _A=True					, _A=None					, _A=True					, ):
									if never_split is None:
														__A     :						Any          =    []
									__A     :						List[Any]          =    do_lower_case
									__A     :						Optional[int]          =    set(_A					)
									__A     :						Tuple          =    tokenize_chinese_chars
									__A     :						List[Any]          =    strip_accents
									__A     :						Dict          =    do_split_on_punc



				def  UpperCAmelCase_		(      self					, _A					, _A=None					):
									__A     :						Union[str, Any]          =    self.never_split.union(set(_A					)					) if never_split else self.never_split
									__A     :						Any          =    self._clean_text(_A					)

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
														__A     :						Any          =    self._tokenize_chinese_chars(_A					)
									# prevents treating the same character with different unicode codepoints as different characters
									__A     :						List[str]          =    unicodedata.normalize('NFC'					, _A					)
									__A     :						Any          =    whitespace_tokenize(_A					)
									__A     :						Dict          =    []
									for token in orig_tokens:
														if token not in never_split:
																			if self.do_lower_case:
																								__A     :						List[Any]          =    token.lower()
																								if self.strip_accents is not False:
																													__A     :						List[str]          =    self._run_strip_accents(_A					)
																			elif self.strip_accents:
																								__A     :						Union[str, Any]          =    self._run_strip_accents(_A					)
														split_tokens.extend(self._run_split_on_punc(_A					, _A					)					)

									__A     :						List[str]          =    whitespace_tokenize(' '.join(_A					)					)
									return output_tokens



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						List[Any]          =    unicodedata.normalize('NFD'					, _A					)
									__A     :						Tuple          =    []
									for char in text:
														__A     :						str          =    unicodedata.category(_A					)
														if cat == "Mn":
																			continue
														output.append(_A					)
									return "".join(_A					)



				def  UpperCAmelCase_		(      self					, _A					, _A=None					):
									if not self.do_split_on_punc or (never_split is not None and text in never_split):
														return [text]
									__A     :						Any          =    list(_A					)
									__A     :						Union[str, Any]          =    0
									__A     :						Union[str, Any]          =    True
									__A     :						Dict          =    []
									while i < len(_A					):
														__A     :						Union[str, Any]          =    chars[i]
														if _is_punctuation(_A					):
																			output.append([char]					)
																			__A     :						Union[str, Any]          =    True
														else:
																			if start_new_word:
																								output.append([]					)
																			__A     :						Dict          =    False
																			output[-1].append(_A					)
														i += 1

									return ["".join(_A					) for x in output]



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Optional[int]          =    []
									for char in text:
														__A     :						Union[str, Any]          =    ord(_A					)
														if self._is_chinese_char(_A					):
																			output.append(' '					)
																			output.append(_A					)
																			output.append(' '					)
														else:
																			output.append(_A					)
									return "".join(_A					)



				def  UpperCAmelCase_		(      self					, _A					):
									# This defines a "chinese character" as anything in the CJK Unicode block:
									#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
									#
									# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
									# despite its name. The modern Korean Hangul alphabet is a different block,
									# as is Japanese Hiragana and Katakana. Those alphabets are used to write
									# space-separated words, so they are not treated specially and handled
									# like the all of the other languages.
									if (
									    (cp >= 0X4_E00 and cp <= 0X9_FFF)
									    or (cp >= 0X3_400 and cp <= 0X4_DBF)  #
									    or (cp >= 0X20_000 and cp <= 0X2A_6DF)  #
									    or (cp >= 0X2A_700 and cp <= 0X2B_73F)  #
									    or (cp >= 0X2B_740 and cp <= 0X2B_81F)  #
									    or (cp >= 0X2B_820 and cp <= 0X2C_EAF)  #
									    or (cp >= 0XF_900 and cp <= 0XF_AFF)
									    or (cp >= 0X2F_800 and cp <= 0X2F_A1F)  #
									):  #
														return True

									return False





				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Dict          =    []
									for char in text:
														__A     :						Tuple          =    ord(_A					)
														if cp == 0 or cp == 0XF_FFD or _is_control(_A					):
																			continue
														if _is_whitespace(_A					):
																			output.append(' '					)
														else:
																			output.append(_A					)
									return "".join(_A					)







def 			_SCREAMING_SNAKE_CASE    (       a  )				->				int:
					__A     :						List[str]          =    set()
					__A     :						Optional[int]          =    word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)  )
										__A     :						Union[str, Any]          =    char
					return pairs







def 			_SCREAMING_SNAKE_CASE    (       a  )				->				List[Any]:
					__A     :						int          =    text.replace('—' ,						'-'  )
					__A     :						Union[str, Any]          =    text.replace('–' ,						'-'  )
					__A     :						str          =    text.replace('―' ,						'-'  )
					__A     :						int          =    text.replace('…' ,						'...'  )
					__A     :						Optional[int]          =    text.replace('´' ,						'\''  )
					__A     :						Dict          =    re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)' ,						r' \1 ' ,						a  )
					__A     :						Optional[int]          =    re.sub(r'\s*\n\s*' ,						' \n ' ,						a  )
					__A     :						List[Any]          =    re.sub(r'[^\S\n]+' ,						' ' ,						a  )
					return text.strip()





class 						_A( snake_case__       ):


				"""simple docstring"""


				UpperCamelCase   :  int    =     VOCAB_FILES_NAMES
				UpperCamelCase   :  Union[str, Any]    =     PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase   :  Any    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase   :  Dict    =     ['''input_ids''', '''attention_mask''']



				def __init__(      self					, _A					, _A					, _A="<unk>"					, **_A					):
									super().__init__(unk_token=_A					, **_A					)

									try:
														import ftfy
														from spacy.lang.en import English

														__A     :						List[str]          =    English()
														__A     :						Tuple          =    _nlp.tokenizer
														__A     :						Dict          =    ftfy.fix_text
									except ImportError:
														logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'					)
														__A     :						List[str]          =    BasicTokenizer(do_lower_case=_A					)
														__A     :						str          =    None

									with open(_A					, encoding='utf-8'					) as vocab_handle:
														__A     :						Union[str, Any]          =    json.load(_A					)
									__A     :						List[Any]          =    {v: k for k, v in self.encoder.items()}
									with open(_A					, encoding='utf-8'					) as merges_handle:
														__A     :						List[str]          =    merges_handle.read().split('\n'					)[1:-1]
									__A     :						List[Any]          =    [tuple(merge.split()					) for merge in merges]
									__A     :						Dict          =    dict(zip(_A					, range(len(_A					)					)					)					)
									__A     :						Any          =    {}



				@property
				def  UpperCAmelCase_		(      self					):
									return True



				@property
				def  UpperCAmelCase_		(      self					):
									return len(self.encoder					)



				def  UpperCAmelCase_		(      self					):
									return dict(self.encoder					, **self.added_tokens_encoder					)



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Optional[int]          =    tuple(token[:-1]					) + (token[-1] + '</w>',)
									if token in self.cache:
														return self.cache[token]
									__A     :						Any          =    get_pairs(_A					)

									if not pairs:
														return token + "</w>"

									while True:
														__A     :						Tuple          =    min(_A					, key=lambda _A					: self.bpe_ranks.get(_A					, float('inf'					)					)					)
														if bigram not in self.bpe_ranks:
																			break
														__A     :						Optional[Any]          =    bigram
														__A     :						Dict          =    []
														__A     :						List[str]          =    0
														while i < len(_A					):
																			try:
																								__A     :						Union[str, Any]          =    word.index(_A					, _A					)
																			except ValueError:
																								new_word.extend(word[i:]					)
																								break
																			else:
																								new_word.extend(word[i:j]					)
																								__A     :						List[str]          =    j

																			if word[i] == first and i < len(_A					) - 1 and word[i + 1] == second:
																								new_word.append(first + second					)
																								i += 2
																			else:
																								new_word.append(word[i]					)
																								i += 1
														__A     :						Union[str, Any]          =    tuple(_A					)
														__A     :						Union[str, Any]          =    new_word
														if len(_A					) == 1:
																			break
														else:
																			__A     :						List[Any]          =    get_pairs(_A					)
									__A     :						Optional[int]          =    ' '.join(_A					)
									if word == "\n  </w>":
														__A     :						Tuple          =    '\n</w>'
									__A     :						List[Any]          =    word
									return word



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Union[str, Any]          =    []
									if self.fix_text is None:
														# Using BERT's BasicTokenizer
														__A     :						Optional[Any]          =    self.nlp.tokenize(_A					)
														for token in text:
																			split_tokens.extend(list(self.bpe(_A					).split(' '					)					)					)
									else:
														# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
														__A     :						Optional[int]          =    self.nlp(text_standardize(self.fix_text(_A					)					)					)
														for token in text:
																			split_tokens.extend(list(self.bpe(token.text.lower()					).split(' '					)					)					)
									return split_tokens



				def  UpperCAmelCase_		(      self					, _A					):
									return self.encoder.get(_A					, self.encoder.get(self.unk_token					)					)



				def  UpperCAmelCase_		(      self					, _A					):
									return self.decoder.get(_A					, self.unk_token					)



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Tuple          =    ''.join(_A					).replace('</w>'					, ' '					).strip()
									return out_string





				def  UpperCAmelCase_		(      self					, _A					, _A = None					):
									if not os.path.isdir(_A					):
														logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
														return
									__A     :						Optional[int]          =    os.path.join(
									    _A					, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
									__A     :						Any          =    os.path.join(
									    _A					, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

									with open(_A					, 'w'					, encoding='utf-8'					) as f:
														f.write(json.dumps(self.encoder					, indent=2					, sort_keys=_A					, ensure_ascii=_A					) + '\n'					)

									__A     :						Optional[Any]          =    0
									with open(_A					, 'w'					, encoding='utf-8'					) as writer:
														writer.write('#version: 0.2\n'					)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					, key=lambda _A					: kv[1]					):
																			if index != token_index:
																								logger.warning(
																								    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																								    ' Please check that the tokenizer is not corrupted!'					)
																								__A     :						Optional[int]          =    token_index
																			writer.write(' '.join(_A					) + '\n'					)
																			index += 1

									return vocab_file, merge_file











import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
				import importlib.resources as pkg_resources
except ImportError:
				# Try backported to PY<37 `importlib_resources`.
				import importlib_resources as pkg_resources

from . import resources


UpperCAmelCase						:       Optional[Any]        =					'''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
UpperCAmelCase						:       Any        =					F"""{BASE_REF_URL}/{__file__}"""
UpperCAmelCase						:       int        =					logging.getLogger(__name__)







def 			_SCREAMING_SNAKE_CASE    (       a  )				->				Tuple[Any, str]:
					__A     :						List[Any]          =    pkg_resources.read_text(a ,						a  )
					return yaml.safe_load(a  ), F"""{BASE_REF_URL}/resources/{resource}"""


UpperCAmelCase  ,			UpperCAmelCase						:       Tuple        =					load_yaml_resource('''readme_structure.yaml''')

UpperCAmelCase						:       List[Any]        =					[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
UpperCAmelCase						:       Union[str, Any]        =					Tuple[dict, List[str], List[str]]





class 						_A:


				"""simple docstring"""


				def __init__(      self					, _A					, _A					, _A = None					, _A = False					):
									__A     :						Dict          =    name
									__A     :						str          =    level
									__A     :						List[str]          =    lines
									__A     :						List[Any]          =    ''
									__A     :						Optional[Any]          =    True
									__A     :						List[Any]          =    {}
									__A     :						Optional[Any]          =    []
									__A     :						Optional[int]          =    []
									if self.lines is not None:
														self.parse(suppress_parsing_errors=_A					)



				def  UpperCAmelCase_		(      self					, _A = False					):
									__A     :						Any          =    ''
									__A     :						Union[str, Any]          =    []
									__A     :						Tuple          =    False
									for line in self.lines:
														if line.strip(' \n'					) == "":
																			continue
														elif line.strip(' \n'					)[:3] == "```":
																			__A     :						int          =    not code_start
														elif line.split()[0] == self.level + "#" and not code_start:
																			if current_sub_level != "":
																								__A     :						Dict          =    Section(_A					, self.level + '#'					, _A					)
																								__A     :						Optional[int]          =    []
																			else:
																								if current_lines != []:
																													self.text += "".join(_A					).strip()
																													if self.text != "" and self.text not in FILLER_TEXT:
																																		__A     :						Union[str, Any]          =    False
																													__A     :						Dict          =    []

																			__A     :						Optional[Any]          =    ' '.join(line.split()[1:]					).strip(' \n'					)
														else:
																			current_lines.append(_A					)
									else:
														if current_sub_level != "":
																			if current_sub_level in self.content:
																								self.parsing_error_list.append(
																								    F"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""					)
																			__A     :						Any          =    Section(_A					, self.level + '#'					, _A					)
														else:
																			if current_lines != []:
																								self.text += "".join(_A					).strip()
																								if self.text != "" and self.text not in FILLER_TEXT:
																													__A     :						str          =    False

									if self.level == "" and not suppress_parsing_errors:
														if self.parsing_error_list != [] or self.parsing_warning_list != []:
																			__A     :						Optional[int]          =    '\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list					)
																			__A     :						List[Any]          =    F"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
																			raise ValueError(_A					)



				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						Any          =    []
									__A     :						Tuple          =    []
									if structure["allow_empty"] is False:
														# If content is expected
														if self.is_empty_text and self.content == {}:
																			# If no content is found, mention it in the error_list
																			error_list.append(F"""Expected some content in section `{self.name}` but it is empty."""					)

									if structure["allow_empty_text"] is False:
														# If some text is expected
														if self.is_empty_text:
																			# If no text is found, mention it in the error_list
																			error_list.append(
																			    F"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""					)
        # Subsections Validation
									if structure["subsections"] is not None:
														# If subsections are expected
														if self.content == {}:
																			# If no subsections are present
																			__A     :						int          =    [subsection['name'] for subsection in structure['subsections']]
																			# Mention the expected values in the error_list
																			error_list.append(
																			    F"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]					)}. Found 'None'."""					)
														else:
																			# If some subsections are present
																			__A     :						Optional[Any]          =    [subsection['name'] for subsection in structure['subsections']]
																			__A     :						str          =    False
																			for idx, name in enumerate(_A					):
																								if name not in self.content:
																													# If the expected subsection is not present
																													error_list.append(F"""Section `{self.name}` is missing subsection: `{name}`."""					)
																													__A     :						Optional[Any]          =    True
																								else:
																													# If the subsection is present, validate subsection, return the result
																													# and concat the errors from subsection to section error_list

																													# Skip sublevel validation if current level is `###`
																													if self.level == "###":
																																		continue
																													else:
																																		__A   ,    __A   ,    __A     :						Optional[int]          =    self.content[name].validate(
																																		    structure['subsections'][idx]					)
																													error_list += subsec_error_list
																													warning_list += subsec_warning_list

																			if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																								for name in self.content:
																													if name not in structure_names:
																																		# If an extra subsection is present
																																		warning_list.append(
																																		    F"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""					)
									if error_list:
														# If there are errors, do not return the dictionary as it is invalid
														return {}, error_list, warning_list
									else:
														return self.to_dict(), error_list, warning_list





				def  UpperCAmelCase_		(      self					):
									return {
									    "name": self.name,
									    "text": self.text,
									    "is_empty_text": self.is_empty_text,
									    "subsections": [value.to_dict() for value in self.content.values()],
									}





class 						_A( snake_case__       ):  # Level 0


				"""simple docstring"""


				def __init__(      self					, _A					, _A					, _A = None					, _A = False					):
									super().__init__(name=_A					, level=''					)  # Not using lines here as we need to use a child class parse
									__A     :						List[str]          =    structure
									__A     :						str          =    -2
									__A     :						Tuple          =    0
									__A     :						Dict          =    lines
									if self.lines is not None:
														self.parse(suppress_parsing_errors=_A					)



				def  UpperCAmelCase_		(      self					):
									if self.structure is None:
														__A   ,    __A   ,    __A     :						List[Any]          =    self._validate(_A					)
									else:
														__A   ,    __A   ,    __A     :						Any          =    self._validate(self.structure					)
									if error_list != [] or warning_list != []:
														__A     :						Optional[Any]          =    '\n'.join(['-\t' + x for x in error_list + warning_list]					)
														__A     :						Union[str, Any]          =    F"""The following issues were found for the README at `{self.name}`:\n""" + errors
														raise ValueError(_A					)



				@classmethod
				def  UpperCAmelCase_		(      cls					, _A					, _A = None					, _A = False					):
									with open(_A					, encoding='utf-8'					) as f:
														__A     :						str          =    f.readlines()
									return cls(_A					, _A					, _A					, suppress_parsing_errors=_A					)



				@classmethod
				def  UpperCAmelCase_		(      cls					, _A					, _A = None					, _A = "root"					, _A = False					):
									__A     :						Any          =    string.split('\n'					)
									return cls(_A					, _A					, _A					, suppress_parsing_errors=_A					)



				def  UpperCAmelCase_		(      self					, _A = False					):
									# Skip Tags
									__A     :						Any          =    0

									for line in self.lines:
														self.yaml_tags_line_count += 1
														if line.strip(' \n'					) == "---":
																			self.tag_count += 1
																			if self.tag_count == 2:
																								break
														line_count += 1
									if self.tag_count == 2:
														__A     :						List[str]          =    self.lines[line_count + 1 :]  # Get the last + 1 th item.
									else:
														__A     :						List[Any]          =    self.lines[self.tag_count :]
									super().parse(suppress_parsing_errors=_A					)



				def __str__(      self					):
									return str(self.to_dict()					)





				def  UpperCAmelCase_		(      self					, _A					):
									__A     :						List[str]          =    []
									__A     :						List[Any]          =    []
									if self.yaml_tags_line_count == 0:
														warning_list.append('Empty YAML markers are present in the README.'					)
									elif self.tag_count == 0:
														warning_list.append('No YAML markers are present in the README.'					)
									elif self.tag_count == 1:
														warning_list.append('Only the start of YAML tags present in the README.'					)
									# Check how many first level sections are present.
									__A     :						Optional[int]          =    len(self.content.keys()					)
									if num_first_level_keys > 1:
														# If more than one, add to the error list, continue
														error_list.append(
														    F"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()					)]					)}. Only one heading is expected. Skipping further validation for this README."""					)
									elif num_first_level_keys < 1:
														# If less than one, append error.
														error_list.append(
														    'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'					)

									else:
														# If one exactly
														__A     :						List[Any]          =    list(self.content.keys()					)[0]  # Get the key
														if start_key.startswith('Dataset Card for'					):  # Check correct start
																			# If the starting is correct, validate all the sections
																			__A   ,    __A   ,    __A     :						List[str]          =    self.content[start_key].validate(
																			    readme_structure['subsections'][0]					)
																			error_list += sec_error_list
																			warning_list += sec_warning_list
														else:
																			# If not found, append error
																			error_list.append(
																			    'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'					)
									if error_list:
														# If there are errors, do not return the dictionary as it is invalid
														return {}, error_list, warning_list
									else:
														return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
				from argparse import ArgumentParser

				UpperCAmelCase						:       Dict        =					ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
				ap.add_argument('''readme_filepath''')
				UpperCAmelCase						:       Tuple        =					ap.parse_args()
				UpperCAmelCase						:       List[str]        =					Path(args.readme_filepath)
				UpperCAmelCase						:       int        =					ReadMe.from_readme(readme_filepath)




