





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


_lowerCamelCase							=     logging.get_logger(__name__)

_lowerCamelCase							=     {'vocab_file': 'vocab.txt'}

_lowerCamelCase							=     {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

_lowerCamelCase							=     {
    'tau/splinter-base': 512,
    'tau/splinter-base-qass': 512,
    'tau/splinter-large': 512,
    'tau/splinter-large-qass': 512,
}

_lowerCamelCase							=     {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}




class 					UpperCamelCase_    (       UpperCamelCase__      ):
							lowerCamelCase_          =   VOCAB_FILES_NAMES
							lowerCamelCase_          =   PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_          =   PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase_          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase_          =   SplinterTokenizer


							def __init__(							self      :int							,			__A      :int=None							,			__A      :List[Any]=None							,			__A      :Any=True							,			__A      :Dict="[UNK]"							,			__A      :int="[SEP]"							,			__A      :Dict="[PAD]"							,			__A      :Union[str, Any]="[CLS]"							,			__A      :List[str]="[MASK]"							,			__A      :Optional[int]="[QUESTION]"							,			__A      :Dict=True							,			__A      :Optional[Any]=None							,			**__A      :List[Any]							,			)				->				Union[str, Any]:



									"""simple docstring"""



									super().__init__(
									    __A							,			tokenizer_file=__A							,			do_lower_case=__A							,			unk_token=__A							,			sep_token=__A							,			pad_token=__A							,			cls_token=__A							,			mask_token=__A							,			tokenize_chinese_chars=__A							,			strip_accents=__A							,			additional_special_tokens=(question_token,)							,			**__A							,			)

									SCREAMING_SNAKE_CASE__								= json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
									if (
									    pre_tok_state.get("""lowercase"""							,			__A						) != do_lower_case
									    or pre_tok_state.get("""strip_accents"""							,			__A						) != strip_accents
									):
											SCREAMING_SNAKE_CASE__								= getattr(__A							,			pre_tok_state.pop("""type"""						)						)
											SCREAMING_SNAKE_CASE__								= do_lower_case
											SCREAMING_SNAKE_CASE__								= strip_accents
											SCREAMING_SNAKE_CASE__								= pre_tok_class(**__A						)

									SCREAMING_SNAKE_CASE__								= do_lower_case


							@property
							def        _snake_case					(							self      :Union[str, Any]						)				->				List[Any]:



									"""simple docstring"""



									return self.convert_tokens_to_ids(self.question_token						)


							def        _snake_case					(							self      :str							,			__A      :List[int]							,			__A      :Optional[List[int]] = None						)				->				List[int]:



									"""simple docstring"""



									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									SCREAMING_SNAKE_CASE__								= [self.cls_token_id]
									SCREAMING_SNAKE_CASE__								= [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								= [self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
									if self.padding_side == "right":
											# Input is question-then-context
											return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
											# Input is context-then-question
											return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


							def        _snake_case					(							self      :str							,			__A      :List[int]							,			__A      :Optional[List[int]] = None						)				->				List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								= [self.cls_token_id]
									SCREAMING_SNAKE_CASE__								= [self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
									if token_ids_a is None:
											return len(cls + token_ids_a + sep						) * [0]

									if self.padding_side == "right":
											# Input is question-then-context
											return len(cls + token_ids_a + question_suffix + sep						) * [0] + len(token_ids_a + sep						) * [1]
									else:
											# Input is context-then-question
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + question_suffix + sep						) * [1]


							def        _snake_case					(							self      :int							,			__A      :str							,			__A      :Optional[str] = None						)				->				Tuple[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self._tokenizer.model.save(__A							,			name=__A						)
									return tuple(__A						)






import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_lowerCamelCase							=     500000

_lowerCamelCase			,    _lowerCamelCase							=     os.path.split(__file__)
_lowerCamelCase							=     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))






@get_duration
def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							datasets.Dataset      ):
		SCREAMING_SNAKE_CASE__								= dataset.select(range(0   ,				len(UpperCamelCase__      )   ,				2      )      )






@get_duration
def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							datasets.Dataset      ):
		SCREAMING_SNAKE_CASE__								= dataset.sort("""numbers"""      )






@get_duration
def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							datasets.Dataset      ):
		SCREAMING_SNAKE_CASE__								= dataset.shuffle()






@get_duration
def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							datasets.Dataset      ):
		SCREAMING_SNAKE_CASE__								= dataset.train_test_split(0.1      )






@get_duration
def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							datasets.Dataset   ,				UpperCamelCase__:							Tuple=10      ):
		for shard_id in range(UpperCamelCase__      ):
				SCREAMING_SNAKE_CASE__								= dataset.shard(UpperCamelCase__   ,				UpperCamelCase__      )






def 			SCREAMING_SNAKE_CASE__							(    ):
		SCREAMING_SNAKE_CASE__								= {"""num examples""": SPEED_TEST_N_EXAMPLES}
		SCREAMING_SNAKE_CASE__								= (select, sort, shuffle, train_test_split, shard)
		with tempfile.TemporaryDirectory() as tmp_dir:
				print("""generating dataset"""      )
				SCREAMING_SNAKE_CASE__								= datasets.Features({"""text""": datasets.Value("""string"""      ), """numbers""": datasets.Value("""float32"""      )}      )
				SCREAMING_SNAKE_CASE__								= generate_example_dataset(
				    os.path.join(UpperCamelCase__   ,				"""dataset.arrow"""      )   ,				UpperCamelCase__   ,				num_examples=UpperCamelCase__      )
				print("""Functions"""      )
				for func in functions:
						print(func.__name__      )
						SCREAMING_SNAKE_CASE__								= func(UpperCamelCase__      )

		with open(UpperCamelCase__   ,				"""wb"""      ) as f:
				f.write(json.dumps(UpperCamelCase__      ).encode("""utf-8"""      )      )


if __name__ == "__main__":  # useful to run the profiler
	benchmark_indices_mapping()