




import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
				from .tokenization_deberta_va import DebertaVaTokenizer
else:
				lowercase_     :    Union[str, Any]      =			None

lowercase_     :    List[str]      =			logging.get_logger(__name__)

lowercase_     :    Any      =			{'vocab_file': 'spm.model', 'tokenizer_file': 'tokenizer.json'}

lowercase_     :    int      =			{
    'vocab_file': {
        'microsoft/deberta-v2-xlarge': 'https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xxlarge': 'https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'
        ),
        'microsoft/deberta-v2-xxlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'
        ),
    }
}

lowercase_     :    Optional[int]      =			{
    'microsoft/deberta-v2-xlarge': 5_12,
    'microsoft/deberta-v2-xxlarge': 5_12,
    'microsoft/deberta-v2-xlarge-mnli': 5_12,
    'microsoft/deberta-v2-xxlarge-mnli': 5_12,
}

lowercase_     :    str      =			{
    'microsoft/deberta-v2-xlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xlarge-mnli': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge-mnli': {'do_lower_case': False},
}






class    __lowerCAmelCase			(	UpperCAmelCase__			):
			snake_case_				:			Union[str, Any]								=					VOCAB_FILES_NAMES
			snake_case_				:			Tuple								=					PRETRAINED_VOCAB_FILES_MAP
			snake_case_				:			Optional[int]								=					PRETRAINED_INIT_CONFIGURATION
			snake_case_				:			List[str]								=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case_				:			int								=					DebertaVaTokenizer







			def __init__(	self   :      Tuple   ,   snake_case__   :      List[Any]=None   ,   snake_case__   :      Union[str, Any]=None   ,   snake_case__   :      Tuple=False   ,   snake_case__   :      Dict=False   ,   snake_case__   :      List[str]="[CLS]"   ,   snake_case__   :      str="[SEP]"   ,   snake_case__   :      Union[str, Any]="[UNK]"   ,   snake_case__   :      str="[SEP]"   ,   snake_case__   :      int="[PAD]"   ,   snake_case__   :      int="[CLS]"   ,   snake_case__   :      List[str]="[MASK]"   ,   **snake_case__   :      List[str]   ,   ):




									"""simple docstring"""


									super().__init__(
									    snake_case__   ,   tokenizer_file=snake_case__   ,   do_lower_case=snake_case__   ,   bos_token=snake_case__   ,   eos_token=snake_case__   ,   unk_token=snake_case__   ,   sep_token=snake_case__   ,   pad_token=snake_case__   ,   cls_token=snake_case__   ,   mask_token=snake_case__   ,   split_by_punct=snake_case__   ,   **snake_case__   ,   )

									_UpperCAmelCase													=					do_lower_case
									_UpperCAmelCase													=					split_by_punct
									_UpperCAmelCase													=					vocab_file
									_UpperCAmelCase													=					False if not self.vocab_file else True







			def UpperCamelCase      (	self   :      Any   ,   snake_case__   :      Any   ,   snake_case__   :      Optional[int]=None				):




									"""simple docstring"""


									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_UpperCAmelCase													=					[self.cls_token_id]
									_UpperCAmelCase													=					[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep







			def UpperCamelCase      (	self   :      Tuple   ,   snake_case__   :      Tuple   ,   snake_case__   :      List[str]=None   ,   snake_case__   :      Dict=False				):




									"""simple docstring"""


									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=snake_case__   ,   token_ids_a=snake_case__   ,   already_has_special_tokens=snake_case__				)

									if token_ids_a is not None:
															return [1] + ([0] * len(snake_case__				)) + [1] + ([0] * len(snake_case__				)) + [1]
									return [1] + ([0] * len(snake_case__				)) + [1]







			def UpperCamelCase      (	self   :      Dict   ,   snake_case__   :      Any   ,   snake_case__   :      Optional[Any]=None				):




									"""simple docstring"""


									_UpperCAmelCase													=					[self.sep_token_id]
									_UpperCAmelCase													=					[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep				) * [0]
									return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      str   ,   snake_case__   :      Optional[str] = None				):




									"""simple docstring"""


									if not self.can_save_slow_tokenizer:
															raise ValueError(
															    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
															    "tokenizer."				)

									if not os.path.isdir(snake_case__				):
															logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
															return
									_UpperCAmelCase													=					os.path.join(
									    snake_case__   ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

									if os.path.abspath(self.vocab_file				) != os.path.abspath(snake_case__				):
															copyfile(self.vocab_file   ,   snake_case__				)

									return (out_vocab_file,)










import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowercase_     :    Union[str, Any]      =			datasets.utils.logging.get_logger(__name__)






@dataclass
class    __lowerCAmelCase			(	datasets.BuilderConfig			):
			snake_case_				:			Optional[datasets.Features]								=					None






class    __lowerCAmelCase			(	datasets.ArrowBasedBuilder			):
			snake_case_				:			str								=					ArrowConfig







			def UpperCamelCase      (	self   :      Optional[int]				):




									"""simple docstring"""


									return datasets.DatasetInfo(features=self.config.features				)







			def UpperCamelCase      (	self   :      str   ,   snake_case__   :      Optional[int]				):




									"""simple docstring"""


									if not self.config.data_files:
															raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""				)
									_UpperCAmelCase													=					dl_manager.download_and_extract(self.config.data_files				)
									if isinstance(snake_case__   ,   (str, list, tuple)				):
															_UpperCAmelCase													=					data_files
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,   gen_kwargs={"files": files}				)]
									_UpperCAmelCase													=					[]
									for split_name, files in data_files.items():
															if isinstance(snake_case__   ,   snake_case__				):
																					_UpperCAmelCase													=					[files]
															# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
															_UpperCAmelCase													=					[dl_manager.iter_files(snake_case__				) for file in files]
															# Infer features is they are stoed in the arrow schema
															if self.info.features is None:
																					for file in itertools.chain.from_iterable(snake_case__				):
																											with open(snake_case__   ,   "rb"				) as f:
																																	_UpperCAmelCase													=					datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case__				).schema				)
																											break
															splits.append(datasets.SplitGenerator(name=snake_case__   ,   gen_kwargs={"files": files}				)				)
									return splits







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      pa.Table				):




									"""simple docstring"""


									if self.info.features is not None:
															# more expensive cast to support nested features with keys in a different order
															# allows str <-> int/float or str to Audio for example
															_UpperCAmelCase													=					table_cast(snake_case__   ,   self.info.features.arrow_schema				)
									return pa_table







			def UpperCamelCase      (	self   :      Optional[Any]   ,   snake_case__   :      int				):




									"""simple docstring"""


									for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__				)				):
															with open(snake_case__   ,   "rb"				) as f:
																					try:
																											for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case__				)				):
																																	_UpperCAmelCase													=					pa.Table.from_batches([record_batch]				)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield F"""{file_idx}_{batch_idx}""", self._cast_table(snake_case__				)
																					except ValueError as e:
																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case__				)}: {e}"""				)
																											raise




