

import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
       import importlib.resources as pkg_resources
except ImportError:
       # Try backported to PY<37 `importlib_resources`.
       import importlib_resources as pkg_resources

from . import resources


lowerCamelCase									= '''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
lowerCamelCase									= f"""{BASE_REF_URL}/{__file__}"""
lowerCamelCase									= logging.getLogger(__name__)




def       UpperCAmelCase__    (      _A				:			str							):





  '''simple docstring'''







  a__  =pkg_resources.read_text(_A     ,				_A							)
  return yaml.safe_load(_A							), F"""{BASE_REF_URL}/resources/{resource}"""


lowerCamelCase					,	lowerCamelCase									= load_yaml_resource('''readme_structure.yaml''')

lowerCamelCase									= [
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
lowerCamelCase									= Tuple[dict, List[str], List[str]]







class 						__magic_name__      :

   '''simple docstring'''




   def __init__(    self,   lowercase_,   lowercase_,   lowercase_ = None,   lowercase_ = False	)							->   Any:






     """simple docstring"""






     a__  =name
     a__  =level
     a__  =lines
     a__  =''''''
     a__  =True
     a__  ={}
     a__  =[]
     a__  =[]
     if self.lines is not None:
       self.parse(suppress_parsing_errors=__lowerCAmelCase	)



   def 			_UpperCAmelCase			(    self,   lowercase_ = False	)							->   Optional[int]:






     """simple docstring"""






     a__  =''''''
     a__  =[]
     a__  =False
     for line in self.lines:
       if line.strip(''' \n'''	) == "":
         continue
       elif line.strip(''' \n'''	)[:3] == "```":
         a__  =not code_start
       elif line.split()[0] == self.level + "#" and not code_start:
         if current_sub_level != "":
           a__  =Section(__lowerCAmelCase,   self.level + '''#''',   __lowerCAmelCase	)
           a__  =[]
         else:
           if current_lines != []:
             self.text += "".join(__lowerCAmelCase	).strip()
             if self.text != "" and self.text not in FILLER_TEXT:
               a__  =False
             a__  =[]

         a__  =''' '''.join(line.split()[1:]	).strip(''' \n'''	)
       else:
         current_lines.append(__lowerCAmelCase	)
     else:
       if current_sub_level != "":
         if current_sub_level in self.content:
           self.parsing_error_list.append(
               F"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""	)
         a__  =Section(__lowerCAmelCase,   self.level + '''#''',   __lowerCAmelCase	)
       else:
         if current_lines != []:
           self.text += "".join(__lowerCAmelCase	).strip()
           if self.text != "" and self.text not in FILLER_TEXT:
             a__  =False

     if self.level == "" and not suppress_parsing_errors:
       if self.parsing_error_list != [] or self.parsing_warning_list != []:
         a__  =a__  ='''\n'''.join('''-\t''' + x for x in self.parsing_error_list + self.parsing_warning_list	)
         a__  =F"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
         raise ValueError(__lowerCAmelCase	)



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Tuple:






     """simple docstring"""






     a__  =[]
     a__  =[]
     if structure["allow_empty"] is False:
       # If content is expected
       if self.is_empty_text and self.content == {}:
         # If no content is found, mention it in the error_list
         error_list.append(F"""Expected some content in section `{self.name}` but it is empty."""	)

     if structure["allow_empty_text"] is False:
       # If some text is expected
       if self.is_empty_text:
         # If no text is found, mention it in the error_list
         error_list.append(
             F"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""	)
        # Subsections Validation
     if structure["subsections"] is not None:
       # If subsections are expected
       if self.content == {}:
         # If no subsections are present
         a__  =[subsection['''name'''] for subsection in structure['''subsections''']]
         # Mention the expected values in the error_list
         error_list.append(
             F"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]	)}. Found 'None'."""	)
       else:
         # If some subsections are present
         a__  =[subsection['''name'''] for subsection in structure['''subsections''']]
         a__  =False
         for idx, name in enumerate(__lowerCAmelCase	):
           if name not in self.content:
             # If the expected subsection is not present
             error_list.append(F"""Section `{self.name}` is missing subsection: `{name}`."""	)
             a__  =True
           else:
             # If the subsection is present, validate subsection, return the result
             # and concat the errors from subsection to section error_list

             # Skip sublevel validation if current level is `###`
             if self.level == "###":
               continue
             else:
               a__,				a__,				a__  =self.content[name].validate(
                   structure['''subsections'''][idx]	)
             error_list += subsec_error_list
             warning_list += subsec_warning_list

         if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
           for name in self.content:
             if name not in structure_names:
               # If an extra subsection is present
               warning_list.append(
                   F"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""	)
     if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
     else:
       return self.to_dict(), error_list, warning_list



   def 			_UpperCAmelCase			(    self	)							->   List[Any]:






     """simple docstring"""






     return {
         "name": self.name,
         "text": self.text,
         "is_empty_text": self.is_empty_text,
         "subsections": [value.to_dict() for value in self.content.values()],
     }







class 						__magic_name__      (							lowerCAmelCase__							):  # Level 0

   '''simple docstring'''




   def __init__(    self,   lowercase_,   lowercase_,   lowercase_ = None,   lowercase_ = False	)							->   str:






     """simple docstring"""






     super().__init__(name=__lowerCAmelCase,   level=''''''	)  # Not using lines here as we need to use a child class parse
     a__  =structure
     a__  =-2
     a__  =0
     a__  =lines
     if self.lines is not None:
       self.parse(suppress_parsing_errors=__lowerCAmelCase	)



   def 			_UpperCAmelCase			(    self	)							->   int:






     """simple docstring"""






     if self.structure is None:
       a__,				a__,				a__  =self._validate(__lowerCAmelCase	)
     else:
       a__,				a__,				a__  =self._validate(self.structure	)
     if error_list != [] or warning_list != []:
       a__  ='''\n'''.join(['''-\t''' + x for x in error_list + warning_list]	)
       a__  =F"""The following issues were found for the README at `{self.name}`:\n""" + errors
       raise ValueError(__lowerCAmelCase	)



   @classmethod
   def 			_UpperCAmelCase			(    cls,   lowercase_,   lowercase_ = None,   lowercase_ = False	)							->   Optional[Any]:






     """simple docstring"""






     with open(__lowerCAmelCase,   encoding='''utf-8'''	) as f:
       a__  =f.readlines()
     return cls(__lowerCAmelCase,   __lowerCAmelCase,   __lowerCAmelCase,   suppress_parsing_errors=__lowerCAmelCase	)



   @classmethod
   def 			_UpperCAmelCase			(    cls,   lowercase_,   lowercase_ = None,   lowercase_ = "root",   lowercase_ = False	)							->   str:






     """simple docstring"""






     a__  =string.split('''\n'''	)
     return cls(__lowerCAmelCase,   __lowerCAmelCase,   __lowerCAmelCase,   suppress_parsing_errors=__lowerCAmelCase	)



   def 			_UpperCAmelCase			(    self,   lowercase_ = False	)							->   List[Any]:






     """simple docstring"""






     a__  =0

     for line in self.lines:
       self.yaml_tags_line_count += 1
       if line.strip(''' \n'''	) == "---":
         self.tag_count += 1
         if self.tag_count == 2:
           break
       line_count += 1
     if self.tag_count == 2:
       a__  =self.lines[line_count + 1 :]  # Get the last + 1 th item.
     else:
       a__  =self.lines[self.tag_count :]
     super().parse(suppress_parsing_errors=__lowerCAmelCase	)



   def __str__(    self	)							->   List[str]:






     """simple docstring"""






     return str(self.to_dict()	)



   def 			_UpperCAmelCase			(    self,   lowercase_	)							->   Any:






     """simple docstring"""






     a__  =[]
     a__  =[]
     if self.yaml_tags_line_count == 0:
       warning_list.append('''Empty YAML markers are present in the README.'''	)
     elif self.tag_count == 0:
       warning_list.append('''No YAML markers are present in the README.'''	)
     elif self.tag_count == 1:
       warning_list.append('''Only the start of YAML tags present in the README.'''	)
     # Check how many first level sections are present.
     a__  =len(self.content.keys()	)
     if num_first_level_keys > 1:
       # If more than one, add to the error list, continue
       error_list.append(
           F"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()	)]	)}. Only one heading is expected. Skipping further validation for this README."""	)
     elif num_first_level_keys < 1:
       # If less than one, append error.
       error_list.append(
           '''The README has no first-level headings. One heading is expected. Skipping further validation for this README.'''	)

     else:
       # If one exactly
       a__  =list(self.content.keys()	)[0]  # Get the key
       if start_key.startswith('''Dataset Card for'''	):  # Check correct start
         # If the starting is correct, validate all the sections
         a__,				a__,				a__  =self.content[start_key].validate(
             readme_structure['''subsections'''][0]	)
         error_list += sec_error_list
         warning_list += sec_warning_list
       else:
         # If not found, append error
         error_list.append(
             '''No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'''	)
     if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
     else:
       return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
       from argparse import ArgumentParser

       lowerCamelCase									= ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
       ap.add_argument('''readme_filepath''')
       lowerCamelCase									= ap.parse_args()
       lowerCamelCase									= Path(args.readme_filepath)
       lowerCamelCase									= ReadMe.from_readme(readme_filepath)



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


lowerCamelCase									= logging.get_logger(__name__)

lowerCamelCase									= {
    '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json''',
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}







class 						__magic_name__      (							lowerCamelCase__							):

   '''simple docstring'''




   lowerCamelCase__			:     List[Any]    =       'blenderbot'
   lowerCamelCase__			:     int    =       ['past_key_values']
   lowerCamelCase__			:     Optional[Any]    =       {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}



   def __init__(    self,   lowercase_=8008,   lowercase_=128,   lowercase_=2,   lowercase_=10240,   lowercase_=32,   lowercase_=24,   lowercase_=10240,   lowercase_=32,   lowercase_=0.0,   lowercase_=0.0,   lowercase_=True,   lowercase_=True,   lowercase_="gelu",   lowercase_=2560,   lowercase_=0.1,   lowercase_=0.0,   lowercase_=0.0,   lowercase_=0.02,   lowercase_=1,   lowercase_=False,   lowercase_=0,   lowercase_=1,   lowercase_=2,   lowercase_=3,   lowercase_=2,   **lowercase_,   )							->   Any:






     """simple docstring"""






     a__  =vocab_size
     a__  =max_position_embeddings
     a__  =d_model
     a__  =encoder_ffn_dim
     a__  =encoder_layers
     a__  =encoder_attention_heads
     a__  =decoder_ffn_dim
     a__  =decoder_layers
     a__  =decoder_attention_heads
     a__  =dropout
     a__  =attention_dropout
     a__  =activation_dropout
     a__  =activation_function
     a__  =init_std
     a__  =encoder_layerdrop
     a__  =decoder_layerdrop
     a__  =use_cache
     a__  =encoder_layers
     a__  =scale_embedding  # scale factor will be sqrt(d_model) if True

     super().__init__(
         pad_token_id=lowercase_,   bos_token_id=lowercase_,   eos_token_id=lowercase_,   is_encoder_decoder=lowercase_,   decoder_start_token_id=lowercase_,   encoder_no_repeat_ngram_size=lowercase_,   forced_eos_token_id=lowercase_,   **lowercase_,   )







class 						__magic_name__      (							lowerCamelCase__							):

   '''simple docstring'''




   @property
   def 			_UpperCAmelCase			(    self	)							->   Mapping[str, Mapping[int, str]]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]	)
       if self.use_past:
         a__  ={0: '''batch'''}
         a__  ={0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
         a__  ={0: '''batch''', 1: '''decoder_sequence'''}
         a__  ={0: '''batch''', 1: '''decoder_sequence'''}
       if self.use_past:
         self.fill_with_past_key_values_(lowercase_,   direction='''inputs'''	)
     elif self.task == "causal-lm":
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]	)
       if self.use_past:
         a__,				a__  =self.num_layers
         for i in range(lowercase_	):
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
     else:
       a__  =OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
               ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
           ]	)

     return common_inputs



   @property
   # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
   def 			_UpperCAmelCase			(    self	)							->   Mapping[str, Mapping[int, str]]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =super().outputs
     else:
       a__  =super(lowercase_,   self	).outputs
       if self.use_past:
         a__,				a__  =self.num_layers
         for i in range(lowercase_	):
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
           a__  ={0: '''batch''', 2: '''past_sequence + sequence'''}
     return common_outputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     # Generate decoder inputs
     a__  =seq_length if not self.use_past else 1
     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     a__  ={F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
     a__  =dict(**lowercase_,   **lowercase_	)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
       else:
         import torch
       a__,				a__  =common_inputs['''input_ids'''].shape
       a__  =common_inputs['''decoder_input_ids'''].shape[1]
       a__,				a__  =self.num_attention_heads
       a__  =(
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       a__  =decoder_seq_length
       a__  =(
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )
       a__  =torch.cat(
           [common_inputs['''decoder_attention_mask'''], torch.ones(lowercase_,   lowercase_	)],   dim=1	)
       a__  =[]
       a__,				a__  =self.num_layers

       for _ in range(lowercase_	):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
                 torch.zeros(lowercase_	),
             )	)
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
       else:
         import torch
       a__,				a__  =common_inputs['''input_ids'''].shape
       a__  =seqlen
       a__,				a__  =self.num_layers
       a__,				a__  =self.num_attention_heads
       a__  =(
           batch,
           num_encoder_attention_heads,
           past_key_values_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       a__  =common_inputs['''attention_mask'''].dtype
       a__  =torch.cat(
           [common_inputs['''attention_mask'''], torch.ones(lowercase_,   lowercase_,   dtype=lowercase_	)],   dim=1	)
       a__  =[
           (torch.zeros(lowercase_	), torch.zeros(lowercase_	)) for _ in range(lowercase_	)
       ]
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     a__  =compute_effective_axis_dimension(
         lowercase_,   fixed_dimension=OnnxConfig.default_fixed_batch,   num_token_to_add=0	)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     a__  =tokenizer.num_special_tokens_to_add(lowercase_	)
     a__  =compute_effective_axis_dimension(
         lowercase_,   fixed_dimension=OnnxConfig.default_fixed_sequence,   num_token_to_add=lowercase_	)

     # Generate dummy inputs according to compute batch and sequence
     a__  =[''' '''.join([tokenizer.unk_token]	) * seq_length] * batch_size
     a__  =dict(tokenizer(lowercase_,   return_tensors=lowercase_	)	)
     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_ = -1,   lowercase_ = -1,   lowercase_ = False,   lowercase_ = None,   )							->   Mapping[str, Any]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =self._generate_dummy_inputs_for_default_and_seqaseq_lm(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)

     elif self.task == "causal-lm":
       a__  =self._generate_dummy_inputs_for_causal_lm(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)
     else:
       a__  =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           lowercase_,   batch_size=lowercase_,   seq_length=lowercase_,   is_pair=lowercase_,   framework=lowercase_	)

     return common_inputs



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_,   lowercase_,   lowercase_	)							->   Optional[int]:






     """simple docstring"""






     if self.task in ["default", "seq2seq-lm"]:
       a__  =super()._flatten_past_key_values_(lowercase_,   lowercase_,   lowercase_,   lowercase_	)
     else:
       a__  =super(lowercase_,   self	)._flatten_past_key_values_(
           lowercase_,   lowercase_,   lowercase_,   lowercase_	)



   def 			_UpperCAmelCase			(    self,   lowercase_,   lowercase_	)							->   Tuple:






     """simple docstring"""






     if direction not in ["inputs", "outputs"]:
       raise ValueError(F"""direction must either be \"inputs\" or \"outputs\", but {direction} was given"""	)

     a__  ='''past_key_values''' if direction == '''inputs''' else '''present'''
     a__,				a__  =self.num_layers

     a__  ='''past_encoder_sequence'''
     a__  ='''past_decoder_sequence''' if direction == '''inputs''' else '''past_decoder_sequence + sequence'''

     for i in range(lowercase_	):
       a__  ={0: '''batch''', 2: decoder_sequence}
       a__  ={0: '''batch''', 2: decoder_sequence}
       a__  ={0: '''batch''', 2: encoder_sequence}
       a__  ={0: '''batch''', 2: encoder_sequence}
