'''simple docstring'''




import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
      import importlib.resources as pkg_resources
except ImportError:
      # Try backported to PY<37 `importlib_resources`.
      import importlib_resources as pkg_resources

from . import resources


__lowercase				   =  '''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
__lowercase				   =  f'{BASE_REF_URL}/{__file__}'
__lowercase				   =  logging.getLogger(__name__)






def     snake_case__							(      _A: str			)   ->     Tuple[Any, str]:

  '''simple docstring'''


  lowerCAmelCase        =  pkg_resources.read_text(_A				,       _A			)
  return yaml.safe_load(_A			), f"{BASE_REF_URL}/resources/{resource}"


__lowercase   ,					__lowercase				   =  load_yaml_resource('''readme_structure.yaml''')

__lowercase				   =  [
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
__lowercase				   =  Tuple[dict, List[str], List[str]]






class     a__:


   '''simple docstring'''






   def __init__(       self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = False):




     """simple docstring"""


     lowerCAmelCase        =  name
     lowerCAmelCase        =  level
     lowerCAmelCase        =  lines
     lowerCAmelCase        =  """"""
     lowerCAmelCase        =  True
     lowerCAmelCase        =  {}
     lowerCAmelCase        =  []
     lowerCAmelCase        =  []
     if self.lines is not None:
       self.parse(suppress_parsing_errors=__lowerCAmelCase)



   def 							a_	(       self	,   __lowerCAmelCase = False):




     """simple docstring"""


     lowerCAmelCase        =  """"""
     lowerCAmelCase        =  []
     lowerCAmelCase        =  False
     for line in self.lines:
       if line.strip(""" \n""") == "":
         continue
       elif line.strip(""" \n""")[:3] == "```":
         lowerCAmelCase        =  not code_start
       elif line.split()[0] == self.level + "#" and not code_start:
         if current_sub_level != "":
           lowerCAmelCase        =  Section(__lowerCAmelCase	,   self.level + """#"""	,   __lowerCAmelCase)
           lowerCAmelCase        =  []
         else:
           if current_lines != []:
             self.text += "".join(__lowerCAmelCase).strip()
             if self.text != "" and self.text not in FILLER_TEXT:
               lowerCAmelCase        =  False
             lowerCAmelCase        =  []

         lowerCAmelCase        =  """ """.join(line.split()[1:]).strip(""" \n""")
       else:
         current_lines.append(__lowerCAmelCase)
     else:
       if current_sub_level != "":
         if current_sub_level in self.content:
           self.parsing_error_list.append(
               f"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.")
         lowerCAmelCase        =  Section(__lowerCAmelCase	,   self.level + """#"""	,   __lowerCAmelCase)
       else:
         if current_lines != []:
           self.text += "".join(__lowerCAmelCase).strip()
           if self.text != "" and self.text not in FILLER_TEXT:
             lowerCAmelCase        =  False

     if self.level == "" and not suppress_parsing_errors:
       if self.parsing_error_list != [] or self.parsing_warning_list != []:
         lowerCAmelCase        =  lowerCAmelCase        =  """\n""".join("""-\t""" + x for x in self.parsing_error_list + self.parsing_warning_list)
         lowerCAmelCase        =  f"The following issues were found while parsing the README at `{self.name}`:\n" + errors
         raise ValueError(__lowerCAmelCase)



   def 							a_	(       self	,   __lowerCAmelCase):




     """simple docstring"""


     lowerCAmelCase        =  []
     lowerCAmelCase        =  []
     if structure["allow_empty"] is False:
       # If content is expected
       if self.is_empty_text and self.content == {}:
         # If no content is found, mention it in the error_list
         error_list.append(f"Expected some content in section `{self.name}` but it is empty.")

     if structure["allow_empty_text"] is False:
       # If some text is expected
       if self.is_empty_text:
         # If no text is found, mention it in the error_list
         error_list.append(
             f"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).")
        # Subsections Validation
     if structure["subsections"] is not None:
       # If subsections are expected
       if self.content == {}:
         # If no subsections are present
         lowerCAmelCase        =  [subsection["""name"""] for subsection in structure["""subsections"""]]
         # Mention the expected values in the error_list
         error_list.append(
             f"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values])}. Found 'None'.")
       else:
         # If some subsections are present
         lowerCAmelCase        =  [subsection["""name"""] for subsection in structure["""subsections"""]]
         lowerCAmelCase        =  False
         for idx, name in enumerate(__lowerCAmelCase):
           if name not in self.content:
             # If the expected subsection is not present
             error_list.append(f"Section `{self.name}` is missing subsection: `{name}`.")
             lowerCAmelCase        =  True
           else:
             # If the subsection is present, validate subsection, return the result
             # and concat the errors from subsection to section error_list

             # Skip sublevel validation if current level is `###`
             if self.level == "###":
               continue
             else:
               lowerCAmelCase ,					lowerCAmelCase ,					lowerCAmelCase        =  self.content[name].validate(
                   structure["""subsections"""][idx])
             error_list += subsec_error_list
             warning_list += subsec_warning_list

         if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
           for name in self.content:
             if name not in structure_names:
               # If an extra subsection is present
               warning_list.append(
                   f"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.")
     if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
     else:
       return self.to_dict(), error_list, warning_list







   def 							a_	(       self):




     """simple docstring"""


     return {
         "name": self.name,
         "text": self.text,
         "is_empty_text": self.is_empty_text,
         "subsections": [value.to_dict() for value in self.content.values()],
     }







class     a__( lowerCAmelCase__     ):  # Level 0


   '''simple docstring'''






   def __init__(       self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = False):




     """simple docstring"""


     super().__init__(name=__lowerCAmelCase	,   level="""""")  # Not using lines here as we need to use a child class parse
     lowerCAmelCase        =  structure
     lowerCAmelCase        =  -2
     lowerCAmelCase        =  0
     lowerCAmelCase        =  lines
     if self.lines is not None:
       self.parse(suppress_parsing_errors=__lowerCAmelCase)



   def 							a_	(       self):




     """simple docstring"""


     if self.structure is None:
       lowerCAmelCase ,					lowerCAmelCase ,					lowerCAmelCase        =  self._validate(__lowerCAmelCase)
     else:
       lowerCAmelCase ,					lowerCAmelCase ,					lowerCAmelCase        =  self._validate(self.structure)
     if error_list != [] or warning_list != []:
       lowerCAmelCase        =  """\n""".join(["""-\t""" + x for x in error_list + warning_list])
       lowerCAmelCase        =  f"The following issues were found for the README at `{self.name}`:\n" + errors
       raise ValueError(__lowerCAmelCase)



   @classmethod
   def 							a_	(       cls	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = False):




     """simple docstring"""


     with open(__lowerCAmelCase	,   encoding="""utf-8""") as f:
       lowerCAmelCase        =  f.readlines()
     return cls(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   suppress_parsing_errors=__lowerCAmelCase)



   @classmethod
   def 							a_	(       cls	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = "root"	,   __lowerCAmelCase = False):




     """simple docstring"""


     lowerCAmelCase        =  string.split("""\n""")
     return cls(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   suppress_parsing_errors=__lowerCAmelCase)



   def 							a_	(       self	,   __lowerCAmelCase = False):




     """simple docstring"""


     lowerCAmelCase        =  0

     for line in self.lines:
       self.yaml_tags_line_count += 1
       if line.strip(""" \n""") == "---":
         self.tag_count += 1
         if self.tag_count == 2:
           break
       line_count += 1
     if self.tag_count == 2:
       lowerCAmelCase        =  self.lines[line_count + 1 :]  # Get the last + 1 th item.
     else:
       lowerCAmelCase        =  self.lines[self.tag_count :]
     super().parse(suppress_parsing_errors=__lowerCAmelCase)



   def __str__(       self):




     """simple docstring"""


     return str(self.to_dict())







   def 							a_	(       self	,   __lowerCAmelCase):




     """simple docstring"""


     lowerCAmelCase        =  []
     lowerCAmelCase        =  []
     if self.yaml_tags_line_count == 0:
       warning_list.append("""Empty YAML markers are present in the README.""")
     elif self.tag_count == 0:
       warning_list.append("""No YAML markers are present in the README.""")
     elif self.tag_count == 1:
       warning_list.append("""Only the start of YAML tags present in the README.""")
     # Check how many first level sections are present.
     lowerCAmelCase        =  len(self.content.keys())
     if num_first_level_keys > 1:
       # If more than one, add to the error list, continue
       error_list.append(
           f"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys())])}. Only one heading is expected. Skipping further validation for this README.")
     elif num_first_level_keys < 1:
       # If less than one, append error.
       error_list.append(
           """The README has no first-level headings. One heading is expected. Skipping further validation for this README.""")

     else:
       # If one exactly
       lowerCAmelCase        =  list(self.content.keys())[0]  # Get the key
       if start_key.startswith("""Dataset Card for"""):  # Check correct start
         # If the starting is correct, validate all the sections
         lowerCAmelCase ,					lowerCAmelCase ,					lowerCAmelCase        =  self.content[start_key].validate(
             readme_structure["""subsections"""][0])
         error_list += sec_error_list
         warning_list += sec_warning_list
       else:
         # If not found, append error
         error_list.append(
             """No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.""")
     if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
     else:
       return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
      from argparse import ArgumentParser

      __lowercase				   =  ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
      ap.add_argument('''readme_filepath''')
      __lowercase				   =  ap.parse_args()
      __lowercase				   =  Path(args.readme_filepath)
      __lowercase				   =  ReadMe.from_readme(readme_filepath)





'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowercase				   =  logging.get_logger(__name__)

__lowercase				   =  {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowercase				   =  {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

__lowercase				   =  {
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}






@lru_cache()
def     snake_case__							(      )   ->     int:

		'''simple docstring'''


		lowerCAmelCase        =  (
		    list(range(ord("""!"""			)				,       ord("""~"""			) + 1			)			) + list(range(ord("""¡"""			)				,       ord("""¬"""			) + 1			)			) + list(range(ord("""®"""			)				,       ord("""ÿ"""			) + 1			)			)
		)
		lowerCAmelCase        =  bs[:]
		lowerCAmelCase        =  0
		for b in range(2**8			):
				if b not in bs:
						bs.append(_A			)
						cs.append(2**8 + n			)
						n += 1
		lowerCAmelCase        =  [chr(_A			) for n in cs]
		return dict(zip(_A				,       _A			)			)






def     snake_case__							(      _A: int			)   ->     int:

		'''simple docstring'''


		lowerCAmelCase        =  set()
		lowerCAmelCase        =  word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)			)
				lowerCAmelCase        =  char
		return pairs






class     a__( lowerCAmelCase__     ):


			'''simple docstring'''






			UpperCAmelCase_      :					Tuple								=	VOCAB_FILES_NAMES
			UpperCAmelCase_      :					List[str]								=	PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_      :					Any								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_      :					str								=	['''input_ids''', '''attention_mask''']



			def __init__(       self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase="replace"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase=None	,   __lowerCAmelCase=False	,   __lowerCAmelCase=False	,   **__lowerCAmelCase	,   ):




					"""simple docstring"""


					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else bos_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else eos_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else unk_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else pad_token
					super().__init__(
					    errors=__lowerCAmelCase	,   unk_token=__lowerCAmelCase	,   bos_token=__lowerCAmelCase	,   eos_token=__lowerCAmelCase	,   pad_token=__lowerCAmelCase	,   add_prefix_space=__lowerCAmelCase	,   add_bos_token=__lowerCAmelCase	,   **__lowerCAmelCase	,   )
					lowerCAmelCase        =  add_bos_token

					with open(__lowerCAmelCase	,   encoding="""utf-8""") as vocab_handle:
							lowerCAmelCase        =  json.load(__lowerCAmelCase)
					lowerCAmelCase        =  {v: k for k, v in self.encoder.items()}
					lowerCAmelCase        =  errors  # how to handle errors in decoding
					lowerCAmelCase        =  bytes_to_unicode()
					lowerCAmelCase        =  {v: k for k, v in self.byte_encoder.items()}
					with open(__lowerCAmelCase	,   encoding="""utf-8""") as merges_handle:
							lowerCAmelCase        =  merges_handle.read().split("""\n""")[1:-1]
					lowerCAmelCase        =  [tuple(merge.split()) for merge in bpe_merges]
					lowerCAmelCase        =  dict(zip(__lowerCAmelCase	,   range(len(__lowerCAmelCase))))
					lowerCAmelCase        =  {}
					lowerCAmelCase        =  add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					lowerCAmelCase        =  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")



			@property
			def 							a_	(       self):




					"""simple docstring"""


					return len(self.encoder)



			def 							a_	(       self):




					"""simple docstring"""


					return dict(self.encoder	,   **self.added_tokens_encoder)



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					if token in self.cache:
							return self.cache[token]
					lowerCAmelCase        =  tuple(__lowerCAmelCase)
					lowerCAmelCase        =  get_pairs(__lowerCAmelCase)

					if not pairs:
							return token

					while True:
							lowerCAmelCase        =  min(__lowerCAmelCase	,   key=lambda __lowerCAmelCase: self.bpe_ranks.get(__lowerCAmelCase	,   float("""inf""")))
							if bigram not in self.bpe_ranks:
									break
							lowerCAmelCase ,					lowerCAmelCase        =  bigram
							lowerCAmelCase        =  []
							lowerCAmelCase        =  0
							while i < len(__lowerCAmelCase):
									try:
											lowerCAmelCase        =  word.index(__lowerCAmelCase	,   __lowerCAmelCase)
									except ValueError:
											new_word.extend(word[i:])
											break
									else:
											new_word.extend(word[i:j])
											lowerCAmelCase        =  j

									if word[i] == first and i < len(__lowerCAmelCase) - 1 and word[i + 1] == second:
											new_word.append(first + second)
											i += 2
									else:
											new_word.append(word[i])
											i += 1
							lowerCAmelCase        =  tuple(__lowerCAmelCase)
							lowerCAmelCase        =  new_word
							if len(__lowerCAmelCase) == 1:
									break
							else:
									lowerCAmelCase        =  get_pairs(__lowerCAmelCase)
					lowerCAmelCase        =  """ """.join(__lowerCAmelCase)
					lowerCAmelCase        =  word
					return word



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase=None):




					"""simple docstring"""


					if self.add_bos_token:
							lowerCAmelCase        =  [self.bos_token_id]
					else:
							lowerCAmelCase        =  []

					lowerCAmelCase        =  bos_token_ids + token_ids_a

					if token_ids_a is None:
							return output

					return output + bos_token_ids + token_ids_a



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  []
					for token in re.findall(self.pat	,   __lowerCAmelCase):
							lowerCAmelCase        =  """""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase).split(""" """))
					return bpe_tokens



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					return self.encoder.get(__lowerCAmelCase	,   self.encoder.get(self.unk_token))



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					return self.decoder.get(__lowerCAmelCase)



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  """""".join(__lowerCAmelCase)
					lowerCAmelCase        =  bytearray([self.byte_decoder[c] for c in text]).decode("""utf-8"""	,   errors=self.errors)
					return text



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase = None):




					"""simple docstring"""


					if not os.path.isdir(__lowerCAmelCase):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory")
							return
					lowerCAmelCase        =  os.path.join(
					    __lowerCAmelCase	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
					lowerCAmelCase        =  os.path.join(
					    __lowerCAmelCase	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

					with open(__lowerCAmelCase	,   """w"""	,   encoding="""utf-8""") as f:
							f.write(json.dumps(self.encoder	,   indent=2	,   sort_keys=__lowerCAmelCase	,   ensure_ascii=__lowerCAmelCase) + """\n""")

					lowerCAmelCase        =  0
					with open(__lowerCAmelCase	,   """w"""	,   encoding="""utf-8""") as writer:
							writer.write("""#version: 0.2\n""")
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,   key=lambda __lowerCAmelCase: kv[1]):
									if index != token_index:
											logger.warning(
											    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
											    """ Please check that the tokenizer is not corrupted!""")
											lowerCAmelCase        =  token_index
									writer.write(""" """.join(__lowerCAmelCase) + """\n""")
									index += 1

					return vocab_file, merge_file



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase=False	,   **__lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  kwargs.pop("""add_prefix_space"""	,   self.add_prefix_space)
					if is_split_into_words or add_prefix_space:
							lowerCAmelCase        =  """ """ + text
					return (text, kwargs)



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase = False	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   ):




					"""simple docstring"""


					lowerCAmelCase        =  super()._decode(
					    token_ids=__lowerCAmelCase	,   skip_special_tokens=__lowerCAmelCase	,   clean_up_tokenization_spaces=__lowerCAmelCase	,   **__lowerCAmelCase	,   )

					if truncate_before_pattern is not None and len(__lowerCAmelCase) > 0:
							lowerCAmelCase        =  self.truncate(__lowerCAmelCase	,   __lowerCAmelCase)

					return decoded_text







			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase):




					"""simple docstring"""


					def find_re(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase):
							lowerCAmelCase        =  pattern.search(__lowerCAmelCase	,   __lowerCAmelCase)
							return m.start() if m else -1

					lowerCAmelCase        =  [re.compile(__lowerCAmelCase	,   re.MULTILINE) for pattern in truncate_before_pattern]

					lowerCAmelCase        =  list(re.finditer("""^print"""	,   __lowerCAmelCase	,   re.MULTILINE))

					if len(__lowerCAmelCase) > 1:
							lowerCAmelCase        =  completion[: prints[1].start()]

					lowerCAmelCase        =  list(re.finditer("""^def"""	,   __lowerCAmelCase	,   re.MULTILINE))

					if len(__lowerCAmelCase) > 1:
							lowerCAmelCase        =  completion[: defs[1].start()]

					lowerCAmelCase        =  0

					lowerCAmelCase        =  [
					    pos for pos in [find_re(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase) for terminal in terminals] if pos != -1
					]

					if len(__lowerCAmelCase) > 0:
							return completion[: min(__lowerCAmelCase)]
					else:
							return completion




