




import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json
@pytest.fixture
def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			Any:
							lowercase__  :	Optional[Any]        =							tmp_path / "file.jsonl"
							lowercase__  :	Dict        =							textwrap.dedent(
							    "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        "    )
							with open(SCREAMING_SNAKE_CASE_    ,"w"    ) as f:
														f.write(SCREAMING_SNAKE_CASE_    )
							return str(SCREAMING_SNAKE_CASE_    )
@pytest.fixture
def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			List[str]:
							lowercase__  :	Any        =							tmp_path / "file_utf16_encoded.jsonl"
							lowercase__  :	Dict        =							textwrap.dedent(
							    "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        "    )
							with open(SCREAMING_SNAKE_CASE_    ,"w"    ,encoding="utf-16"    ) as f:
														f.write(SCREAMING_SNAKE_CASE_    )
							return str(SCREAMING_SNAKE_CASE_    )
@pytest.fixture
def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			str:
							lowercase__  :	Dict        =							tmp_path / "file_with_list_of_dicts.json"
							lowercase__  :	Union[str, Any]        =							textwrap.dedent(
							    "\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        "    )
							with open(SCREAMING_SNAKE_CASE_    ,"w"    ) as f:
														f.write(SCREAMING_SNAKE_CASE_    )
							return str(SCREAMING_SNAKE_CASE_    )
@pytest.fixture
def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			List[Any]:
							lowercase__  :	Optional[Any]        =							tmp_path / "file_with_list_of_dicts_field.json"
							lowercase__  :	str        =							textwrap.dedent(
							    "\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        "    )
							with open(SCREAMING_SNAKE_CASE_    ,"w"    ) as f:
														f.write(SCREAMING_SNAKE_CASE_    )
							return str(SCREAMING_SNAKE_CASE_    )
@pytest.mark.parametrize(
    "file_fixture, config_kwargs"    ,[
        ("jsonl_file", {}),
        ("jsonl_file_utf16_encoded", {"encoding": "utf-16"}),
        ("json_file_with_list_of_dicts", {}),
        ("json_file_with_list_of_dicts_field", {"field": "field3"}),
    ]    ,)
def  snake_case_					(		SCREAMING_SNAKE_CASE_    ,SCREAMING_SNAKE_CASE_    ,SCREAMING_SNAKE_CASE_    )      ->			Any:
							lowercase__  :	Optional[Any]        =							Json(**SCREAMING_SNAKE_CASE_    )
							lowercase__  :	List[Any]        =							json._generate_tables([[request.getfixturevalue(SCREAMING_SNAKE_CASE_    )]]    )
							lowercase__  :	List[Any]        =							pa.concat_tables([table for _, table in generator]    )
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    "file_fixture, config_kwargs"    ,[
        (
            "jsonl_file",
            {"features": Features({"col_1": Value("int64"    ), "col_2": Value("int64"    ), "missing_col": Value("string"    )}    )},
        ),
        (
            "json_file_with_list_of_dicts",
            {"features": Features({"col_1": Value("int64"    ), "col_2": Value("int64"    ), "missing_col": Value("string"    )}    )},
        ),
        (
            "json_file_with_list_of_dicts_field",
            {
                "field": "field3",
                "features": Features(
                    {"col_1": Value("int64"    ), "col_2": Value("int64"    ), "missing_col": Value("string"    )}    ),
            },
        ),
    ]    ,)
def  snake_case_					(		SCREAMING_SNAKE_CASE_    ,SCREAMING_SNAKE_CASE_    ,SCREAMING_SNAKE_CASE_    )      ->			Optional[Any]:
							lowercase__  :	Any        =							Json(**SCREAMING_SNAKE_CASE_    )
							lowercase__  :	int        =							json._generate_tables([[request.getfixturevalue(SCREAMING_SNAKE_CASE_    )]]    )
							lowercase__  :	Tuple        =							pa.concat_tables([table for _, table in generator]    )
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}





import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
						from .tokenization_deberta_va import DebertaVaTokenizer
else:
						__a  :		Optional[Any]		     =   None

__a  :		List[str]		     =   logging.get_logger(__name__)

__a  :		Dict		     =   {'''vocab_file''': '''spm.model''', '''tokenizer_file''': '''tokenizer.json'''}

__a  :		int		     =   {
    '''vocab_file''': {
        '''microsoft/deberta-v2-xlarge''': '''https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xxlarge''': '''https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'''
        ),
        '''microsoft/deberta-v2-xxlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'''
        ),
    }
}

__a  :		Any		     =   {
    '''microsoft/deberta-v2-xlarge''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge''': 5_1_2,
    '''microsoft/deberta-v2-xlarge-mnli''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge-mnli''': 5_1_2,
}

__a  :		Union[str, Any]		     =   {
    '''microsoft/deberta-v2-xlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xlarge-mnli''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge-mnli''': {'''do_lower_case''': False},
}







class 				UpperCAmelCase(					snake_case_					):



	"""simple docstring"""






	a     : List[Any]	  =					VOCAB_FILES_NAMES
	a     : Optional[int]	  =					PRETRAINED_VOCAB_FILES_MAP
	a     : int	  =					PRETRAINED_INIT_CONFIGURATION
	a     : List[str]	  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     : int	  =					DebertaVaTokenizer

	def __init__(					self	,				lowerCamelCase=None	,				lowerCamelCase=None	,				lowerCamelCase=False	,				lowerCamelCase=False	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[UNK]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[PAD]"	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[MASK]"	,				**lowerCamelCase	,				) ->      None:





								"""simple docstring"""


								super().__init__(
								    lowerCamelCase	,				tokenizer_file=lowerCamelCase	,				do_lower_case=lowerCamelCase	,				bos_token=lowerCamelCase	,				eos_token=lowerCamelCase	,				unk_token=lowerCamelCase	,				sep_token=lowerCamelCase	,				pad_token=lowerCamelCase	,				cls_token=lowerCamelCase	,				mask_token=lowerCamelCase	,				split_by_punct=lowerCamelCase	,				**lowerCamelCase	,				)

								lowercase__  :	Optional[int]        =							do_lower_case
								lowercase__  :	Optional[Any]        =							split_by_punct
								lowercase__  :	Tuple        =							vocab_file
								lowercase__  :	Optional[int]        =							False if not self.vocab_file else True

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      List[Any]:





								"""simple docstring"""


								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowercase__  :	Union[str, Any]        =							[self.cls_token_id]
								lowercase__  :	Union[str, Any]        =							[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None	,				lowerCamelCase=False    ) ->      Union[str, Any]:





								"""simple docstring"""


								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCamelCase	,				token_ids_a=lowerCamelCase	,				already_has_special_tokens=lowerCamelCase    )

								if token_ids_a is not None:
															return [1] + ([0] * len(lowerCamelCase    )) + [1] + ([0] * len(lowerCamelCase    )) + [1]
								return [1] + ([0] * len(lowerCamelCase    )) + [1]

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      Optional[int]:





								"""simple docstring"""


								lowercase__  :	Any        =							[self.sep_token_id]
								lowercase__  :	List[str]        =							[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      Tuple[str]:





								"""simple docstring"""


								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
															    "tokenizer."    )

								if not os.path.isdir(lowerCamelCase    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
															return
								lowercase__  :	int        =							os.path.join(
								    lowerCamelCase	,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

								if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase    ):
															copyfile(self.vocab_file	,				lowerCamelCase    )

								return (out_vocab_file,)