
'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCamelCase_            =    logging.get_logger(__name__)

UpperCamelCase_            =    {"""vocab_file""": """vocab.txt"""}

UpperCamelCase_            =    {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

UpperCamelCase_            =    {
    """tau/splinter-base""": 5_12,
    """tau/splinter-base-qass""": 5_12,
    """tau/splinter-large""": 5_12,
    """tau/splinter-large-qass""": 5_12,
}

UpperCamelCase_            =    {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}


class 					a_					(_a      ):
       __lowerCAmelCase      :			List[Any]      = VOCAB_FILES_NAMES
       __lowerCAmelCase      :			Optional[int]      = PRETRAINED_VOCAB_FILES_MAP
       __lowerCAmelCase      :			str      = PRETRAINED_INIT_CONFIGURATION
       __lowerCAmelCase      :			List[str]      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowerCAmelCase      :			int      = SplinterTokenizer




       def __init__(       self				,				snake_case_=None				,				snake_case_=None				,				snake_case_=True				,				snake_case_="[UNK]"				,				snake_case_="[SEP]"				,				snake_case_="[PAD]"				,				snake_case_="[CLS]"				,				snake_case_="[MASK]"				,				snake_case_="[QUESTION]"				,				snake_case_=True				,				snake_case_=None				,				**snake_case_				,				):
         super().__init__(
             snake_case_				,				tokenizer_file=snake_case_				,				do_lower_case=snake_case_				,				unk_token=snake_case_				,				sep_token=snake_case_				,				pad_token=snake_case_				,				cls_token=snake_case_				,				mask_token=snake_case_				,				tokenize_chinese_chars=snake_case_				,				strip_accents=snake_case_				,				additional_special_tokens=(question_token,)				,				**snake_case_				,				)

         _lowerCAmelCase    :					Tuple							=   json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
         if (
             pre_tok_state.get("""lowercase"""				,				snake_case_					) != do_lower_case
             or pre_tok_state.get("""strip_accents"""				,				snake_case_					) != strip_accents
         ):
           _lowerCAmelCase    :					Union[str, Any]							=   getattr(snake_case_				,				pre_tok_state.pop("""type"""					)					)
           _lowerCAmelCase    :					Any							=   do_lower_case
           _lowerCAmelCase    :					Optional[int]							=   strip_accents
           _lowerCAmelCase    :					List[str]							=   pre_tok_class(**snake_case_					)

         _lowerCAmelCase    :					Optional[int]							=   do_lower_case




       @property
       def 							__UpperCamelCase    (       self					):
         return self.convert_tokens_to_ids(self.question_token					)




       def 							__UpperCamelCase    (       self				,				snake_case_				,				snake_case_ = None					):
         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         _lowerCAmelCase    :					Dict							=   [self.cls_token_id]
         _lowerCAmelCase    :					str							=   [self.sep_token_id]
         _lowerCAmelCase    :					List[str]							=   [self.question_token_id] + [self.convert_tokens_to_ids("""."""					)]
         if self.padding_side == "right":
           # Input is question-then-context
           return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
           # Input is context-then-question
           return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




       def 							__UpperCamelCase    (       self				,				snake_case_				,				snake_case_ = None					):
         _lowerCAmelCase    :					List[Any]							=   [self.sep_token_id]
         _lowerCAmelCase    :					Dict							=   [self.cls_token_id]
         _lowerCAmelCase    :					List[str]							=   [self.question_token_id] + [self.convert_tokens_to_ids("""."""					)]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep					) * [0]

         if self.padding_side == "right":
           # Input is question-then-context
           return len(cls + token_ids_a + question_suffix + sep					) * [0] + len(token_ids_a + sep					) * [1]
         else:
           # Input is context-then-question
           return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + question_suffix + sep					) * [1]







       def 							__UpperCamelCase    (       self				,				snake_case_				,				snake_case_ = None					):
         _lowerCAmelCase    :					Tuple							=   self._tokenizer.model.save(snake_case_				,				name=snake_case_					)
         return tuple(snake_case_					)




'''simple docstring'''

import platform
from argparse import ArgumentParser

import huggingface_hub
import pandas
import pyarrow

from datasets import __version__ as version
from datasets.commands import BaseDatasetsCLICommand


def 		_UpperCAmelCase			(					_lowerCamelCase  :    List[Any]						)      ->  Optional[Any]:
  return EnvironmentCommand()



class 					a_					(_a      ):




       @staticmethod
       def 							__UpperCamelCase    (       snake_case_					):
         _lowerCAmelCase    :					Optional[Any]							=   parser.add_parser("""env"""				,				help="""Print relevant system environment info."""					)
         download_parser.set_defaults(func=snake_case_					)




       def 							__UpperCamelCase    (       self					):
         _lowerCAmelCase    :					Optional[Any]							=   {
             """`datasets` version""": version,
             """Platform""": platform.platform(),
             """Python version""": platform.python_version(),
             """Huggingface_hub version""": huggingface_hub.__version__,
             """PyArrow version""": pyarrow.__version__,
             """Pandas version""": pandas.__version__,
         }

         print("""\nCopy-and-paste the text below in your GitHub issue.\n"""					)
         print(self.format_dict(snake_case_					)					)

         return info







       @staticmethod
       def 							__UpperCamelCase    (       snake_case_					):
         return "\n".join([f'- {prop}: {val}' for prop, val in d.items()]					) + "\n"


