






import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata


def __A   (				__lowerCAmelCase     )->  str:





						"""simple docstring"""



						_UpperCAmelCase     =      min(re.search('^ +'			,		__lowerCAmelCase     ).end() if t.startswith(' '     ) else 0 for t in string.splitlines()     )
						return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(__lowerCAmelCase     )]     )


_a				     =  '''\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
'''

_a				     =  '''\
---
---
# Begin of markdown

Some cool dataset card
'''


_a				     =  '''\
# Begin of markdown

Some cool dataset card
'''





class __lowerCamelCase			(				unittest.TestCase):





	"""simple docstring"""






	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCAmelCase     =      Path(UpperCAmelCase   ) / 'README.md'
													with open(UpperCAmelCase	,   'w+'   ) as readme_file:
																			readme_file.write(UpperCAmelCase   )
													_UpperCAmelCase     =      DatasetMetadata.from_readme(UpperCAmelCase   )
													self.assertDictEqual(UpperCAmelCase	,   {'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}   )

													with open(UpperCAmelCase	,   'w+'   ) as readme_file:
																			readme_file.write(UpperCAmelCase   )
													_UpperCAmelCase     =      DatasetMetadata.from_readme(UpperCAmelCase   )
													self.assertDictEqual(UpperCAmelCase	,   {}   )

													with open(UpperCAmelCase	,   'w+'   ) as readme_file:
																			readme_file.write(UpperCAmelCase   )
													_UpperCAmelCase     =      DatasetMetadata.from_readme(UpperCAmelCase   )
													self.assertEqual(UpperCAmelCase	,   {}   )

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      _dedent(
							    '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '   )
							assert DatasetMetadata.from_yaml_string(UpperCAmelCase   )

							_UpperCAmelCase     =      _dedent(
							    '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '   )
							with self.assertRaises(UpperCAmelCase   ):
													DatasetMetadata.from_yaml_string(UpperCAmelCase   )

							_UpperCAmelCase     =      _dedent(
							    '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '   )
							assert DatasetMetadata.from_yaml_string(UpperCAmelCase   )








import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir





@dataclass
class __lowerCamelCase			:





	"""simple docstring"""






	UpperCamelCase__								=				42





class __lowerCamelCase			(				unittest.TestCase):





	"""simple docstring"""






	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      get_tests_dir('fixtures/test_sentencepiece.model'   )
							_UpperCAmelCase     =      get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'   )

							_UpperCAmelCase     =      FakeOriginalTokenizer(vocab_file=UpperCAmelCase   )

							with warnings.catch_warnings(record=UpperCAmelCase   ) as w:
													_UpperCAmelCase     =      SpmConverter(UpperCAmelCase   )
							self.assertEqual(len(UpperCAmelCase   )	,   0   )

							_UpperCAmelCase     =      FakeOriginalTokenizer(vocab_file=UpperCAmelCase   )

							with warnings.catch_warnings(record=UpperCAmelCase   ) as w:
													_UpperCAmelCase     =      SpmConverter(UpperCAmelCase   )
							self.assertEqual(len(UpperCAmelCase   )	,   1   )

							self.assertIn(
							    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
							    ' which is not implemented in the fast tokenizers.'	,   str(w[0].message   )	,   )
