




import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

UpperCamelCase      			=     logging.get_logger(__name__)

UpperCamelCase      			=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

UpperCamelCase      			=     {
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

UpperCamelCase      			=     {
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

UpperCamelCase      			=     {
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}



def    lowerCamelCase_   (  )							->	Tuple:
		__A							:   Optional[int]	  =				(
		    list(range(ord("!"    ) ,     ord("~"    ) + 1    )    ) + list(range(ord("¡"    ) ,     ord("¬"    ) + 1    )    ) + list(range(ord("®"    ) ,     ord("ÿ"    ) + 1    )    )
		)
		__A							:   Tuple	  =				bs[:]
		__A							:   Union[str, Any]	  =				0
		for b in range(2**8    ):
				if b not in bs:
						bs.append(SCREAMING_SNAKE_CASE__    )
						cs.append(2**8 + n    )
						n += 1
		__A							:   List[Any]	  =				[chr(SCREAMING_SNAKE_CASE__    ) for n in cs]
		return dict(zip(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )    )



def    lowerCamelCase_   (  _lowercase    )							->	str:
		__A							:   Tuple	  =				set()
		__A							:   str	  =				word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				__A							:   str	  =				char
		return pairs





class _a       (	UpperCamelCase_ ):

	'''simple docstring'''



	lowerCamelCase_   :      Any       =	VOCAB_FILES_NAMES
	lowerCamelCase_   :      Any       =	PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase_   :      str       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase_   :      str       =	['input_ids', 'attention_mask', 'token_type_ids']

	def __init__(       self							,			__UpperCAmelCase							,			__UpperCAmelCase							,			__UpperCAmelCase="replace"							,			__UpperCAmelCase="[CLS]"							,			__UpperCAmelCase="[SEP]"							,			__UpperCAmelCase="[SEP]"							,			__UpperCAmelCase="[CLS]"							,			__UpperCAmelCase="[UNK]"							,			__UpperCAmelCase="[PAD]"							,			__UpperCAmelCase="[MASK]"							,			__UpperCAmelCase=False							,			__UpperCAmelCase=False							,			**__UpperCAmelCase							,			):
			__A							:   Union[str, Any]	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else bos_token
			__A							:   int	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else eos_token
			__A							:   List[str]	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else sep_token
			__A							:   Optional[Any]	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else cls_token
			__A							:   Optional[int]	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else unk_token
			__A							:   Optional[Any]	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else pad_token

			# Mask token behave like a normal word, i.e. include the space before it
			__A							:   Dict	  =				AddedToken(lowercase_							,			lstrip=lowercase_							,			rstrip=lowercase_   ) if isinstance(lowercase_							,			lowercase_   ) else mask_token

			super().__init__(
			    errors=lowercase_							,			bos_token=lowercase_							,			eos_token=lowercase_							,			unk_token=lowercase_							,			sep_token=lowercase_							,			cls_token=lowercase_							,			pad_token=lowercase_							,			mask_token=lowercase_							,			add_prefix_space=lowercase_							,			add_bos_token=lowercase_							,			**lowercase_							,			)
			__A							:   int	  =				add_bos_token

			with open(lowercase_							,			encoding="utf-8"   ) as vocab_handle:
					__A							:   List[str]	  =				json.load(lowercase_   )
			__A							:   Any	  =				{v: k for k, v in self.encoder.items()}
			__A							:   List[str]	  =				errors  # how to handle errors in decoding
			__A							:   Union[str, Any]	  =				bytes_to_unicode()
			__A							:   List[Any]	  =				{v: k for k, v in self.byte_encoder.items()}
			with open(lowercase_							,			encoding="utf-8"   ) as merges_handle:
					__A							:   Optional[Any]	  =				merges_handle.read().split("\n"   )[1:-1]
			__A							:   Optional[Any]	  =				[tuple(merge.split()   ) for merge in bpe_merges]
			__A							:   Tuple	  =				dict(zip(lowercase_							,			range(len(lowercase_   )   )   )   )
			__A							:   int	  =				{}
			__A							:   List[str]	  =				add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			__A							:   Tuple	  =				re.compile(r"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"   )

	@property
	# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
	def 					__UpperCAmelCase(       self   ):
			return len(self.encoder   )

	def 					__UpperCAmelCase(       self   ):
			return dict(self.encoder							,			**self.added_tokens_encoder   )

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):




			if token in self.cache:
					return self.cache[token]
			__A							:   Optional[int]	  =				tuple(lowercase_   )
			__A							:   Optional[int]	  =				get_pairs(lowercase_   )

			if not pairs:
					return token

			while True:
					__A							:   Any	  =				min(lowercase_							,			key=lambda __UpperCAmelCase   : self.bpe_ranks.get(lowercase_							,			float("inf"   )   )   )
					if bigram not in self.bpe_ranks:
							break
					__A							:   int	  =				bigram
					__A							:   Any	  =				[]
					__A							:   int	  =				0
					while i < len(lowercase_   ):
							try:
									__A							:   Optional[Any]	  =				word.index(lowercase_							,			lowercase_   )
							except ValueError:
									new_word.extend(word[i:]   )
									break
							else:
									new_word.extend(word[i:j]   )
									__A							:   Optional[Any]	  =				j

							if word[i] == first and i < len(lowercase_   ) - 1 and word[i + 1] == second:
									new_word.append(first + second   )
									i += 2
							else:
									new_word.append(word[i]   )
									i += 1
					__A							:   List[str]	  =				tuple(lowercase_   )
					__A							:   Optional[Any]	  =				new_word
					if len(lowercase_   ) == 1:
							break
					else:
							__A							:   List[Any]	  =				get_pairs(lowercase_   )
			__A							:   Union[str, Any]	  =				' '.join(lowercase_   )
			__A							:   Tuple	  =				word
			return word

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase = None   ):




			if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			__A							:   List[str]	  =				[self.cls_token_id]
			__A							:   Dict	  =				[self.sep_token_id]
			return cls + token_ids_a + sep + token_ids_a + sep

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase = None							,			__UpperCAmelCase = False   ):




			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=lowercase_							,			token_ids_a=lowercase_							,			already_has_special_tokens=lowercase_   )

			if token_ids_a is None:
					return [1] + ([0] * len(lowercase_   )) + [1]
			return [1] + ([0] * len(lowercase_   )) + [1] + ([0] * len(lowercase_   )) + [1]

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase = None   ):
			__A							:   str	  =				[self.sep_token_id]
			__A							:   Optional[int]	  =				[self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep   ) * [0]
			return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):
			__A							:   List[Any]	  =				[]
			for token in re.findall(self.pat							,			lowercase_   ):
					__A							:   int	  =				''.join(
					    self.byte_encoder[b] for b in token.encode("utf-8"   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_   ).split(" "   )   )
			return bpe_tokens

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):
			return self.encoder.get(lowercase_							,			self.encoder.get(self.unk_token   )   )

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):
			return self.decoder.get(lowercase_   )

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):
			__A							:   int	  =				''.join(lowercase_   )
			__A							:   int	  =				bytearray([self.byte_decoder[c] for c in text]   ).decode("utf-8"							,			errors=self.errors   )
			return text

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase = None   ):




			if not os.path.isdir(lowercase_   ):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"   )
					return
			__A							:   Dict	  =				os.path.join(
			    lowercase_							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
			__A							:   Optional[int]	  =				os.path.join(
			    lowercase_							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

			with open(lowercase_							,			"w"							,			encoding="utf-8"   ) as f:
					f.write(json.dumps(self.encoder							,			indent=2							,			sort_keys=lowercase_							,			ensure_ascii=lowercase_   ) + "\n"   )

			__A							:   Union[str, Any]	  =				0
			with open(lowercase_							,			"w"							,			encoding="utf-8"   ) as writer:
					writer.write("#version: 0.2\n"   )
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,			key=lambda __UpperCAmelCase   : kv[1]   ):
							if index != token_index:
									logger.warning(
									    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
									    " Please check that the tokenizer is not corrupted!"   )
									__A							:   Dict	  =				token_index
							writer.write(" ".join(lowercase_   ) + "\n"   )
							index += 1

			return vocab_file, merge_file

	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase							,			__UpperCAmelCase=False							,			**__UpperCAmelCase   ):
			__A							:   Tuple	  =				kwargs.pop("add_prefix_space"							,			self.add_prefix_space   )
			if (is_split_into_words or add_prefix_space) and (len(lowercase_   ) > 0 and not text[0].isspace()):
					__A							:   Tuple	  =				' ' + text
			return (text, kwargs)







	def 					__UpperCAmelCase(       self							,			__UpperCAmelCase   ):
			__A							:   Optional[int]	  =				[]
			for is_user, text in conversation.iter_texts():
					input_ids.extend(self.encode(lowercase_							,			add_special_tokens=lowercase_   ) + [self.eos_token_id]   )
			if len(lowercase_   ) > self.model_max_length:
					__A							:   str	  =				input_ids[-self.model_max_length :]
			return input_ids






import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata



def    lowerCamelCase_   (  _lowercase    )							->	str:
		__A							:   Optional[int]	  =				min(re.search("^ +" ,     _lowercase    ).end() if t.startswith(" "    ) else 0 for t in string.splitlines()    )
		return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(_lowercase    )]    )


UpperCamelCase      			=     '\\n---\nlanguage:\n- zh\n- en\ntask_ids:\n- sentiment-classification\n---\n# Begin of markdown\n\nSome cool dataset card\n'

UpperCamelCase      			=     '\\n---\n---\n# Begin of markdown\n\nSome cool dataset card\n'


UpperCamelCase      			=     '\\n# Begin of markdown\n\nSome cool dataset card\n'





class _a       (	unittest.TestCase ):

	'''simple docstring'''



	def 					__UpperCAmelCase(       self   ):
			with tempfile.TemporaryDirectory() as tmp_dir:
					__A							:   List[Any]	  =				Path(__UpperCAmelCase   ) / "README.md"
					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Tuple	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertDictEqual(__UpperCAmelCase							,			{"language": ["zh", "en"], "task_ids": ["sentiment-classification"]}   )

					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Any	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertDictEqual(__UpperCAmelCase							,			{}   )

					with open(__UpperCAmelCase							,			"w+"   ) as readme_file:
							readme_file.write(__UpperCAmelCase   )
					__A							:   Union[str, Any]	  =				DatasetMetadata.from_readme(__UpperCAmelCase   )
					self.assertEqual(__UpperCAmelCase							,			{}   )







	def 					__UpperCAmelCase(       self   ):
			__A							:   List[str]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            "   )
			assert DatasetMetadata.from_yaml_string(__UpperCAmelCase   )

			__A							:   List[Any]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            "   )
			with self.assertRaises(__UpperCAmelCase   ):
					DatasetMetadata.from_yaml_string(__UpperCAmelCase   )

			__A							:   Optional[int]	  =				_dedent(
			    "\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the \"Researcher\") has requested permission to use the ImageNet database (the \"Database\") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            "   )
			assert DatasetMetadata.from_yaml_string(__UpperCAmelCase   )





