




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


a_										=       logging.get_logger(__name__)

a_										=       {"vocab_file": "vocab.txt"}

a_										=       {
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

a_										=       {
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

a_										=       {
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}

class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

      """simple docstring"""

      lowerCAmelCase__     :	List[Any]      =   VOCAB_FILES_NAMES
      lowerCAmelCase__     :	Optional[Any]      =   PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__     :	Any      =   PRETRAINED_INIT_CONFIGURATION
      lowerCAmelCase__     :	Any      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__     :	List[str]      =   SplinterTokenizer







      def __init__(       self   :	Dict	,    snake_case   :	Dict=None	,    snake_case   :	str=None	,    snake_case   :	Union[str, Any]=True	,    snake_case   :	Union[str, Any]="[UNK]"	,    snake_case   :	List[str]="[SEP]"	,    snake_case   :	List[Any]="[PAD]"	,    snake_case   :	Dict="[CLS]"	,    snake_case   :	Optional[int]="[MASK]"	,    snake_case   :	List[Any]="[QUESTION]"	,    snake_case   :	Union[str, Any]=True	,    snake_case   :	List[str]=None	,    **snake_case   :	Dict	,    ):
          super().__init__(
              snake_case	,    tokenizer_file=snake_case	,    do_lower_case=snake_case	,    unk_token=snake_case	,    sep_token=snake_case	,    pad_token=snake_case	,    cls_token=snake_case	,    mask_token=snake_case	,    tokenize_chinese_chars=snake_case	,    strip_accents=snake_case	,    additional_special_tokens=(question_token,)	,    **snake_case	,    )

          __UpperCamelCase											=  json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
          if (
              pre_tok_state.get('''lowercase'''	,    snake_case			) != do_lower_case
              or pre_tok_state.get('''strip_accents'''	,    snake_case			) != strip_accents
          ):
              __UpperCamelCase											=  getattr(snake_case	,    pre_tok_state.pop('''type'''			)			)
              __UpperCamelCase											=  do_lower_case
              __UpperCamelCase											=  strip_accents
              __UpperCamelCase											=  pre_tok_class(**snake_case			)

          __UpperCamelCase											=  do_lower_case







      @property
      def  snake_case		(       self   :	int			):
          return self.convert_tokens_to_ids(self.question_token			)







      def  snake_case		(       self   :	Union[str, Any]	,    snake_case   :	List[int]	,    snake_case   :	Optional[List[int]] = None			):
          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          __UpperCamelCase											=  [self.cls_token_id]
          __UpperCamelCase											=  [self.sep_token_id]
          __UpperCamelCase											=  [self.question_token_id] + [self.convert_tokens_to_ids('''.'''			)]
          if self.padding_side == "right":
              # Input is question-then-context
              return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
          else:
              # Input is context-then-question
              return cls + token_ids_a + sep + token_ids_a + question_suffix + sep







      def  snake_case		(       self   :	Any	,    snake_case   :	List[int]	,    snake_case   :	Optional[List[int]] = None			):
          __UpperCamelCase											=  [self.sep_token_id]
          __UpperCamelCase											=  [self.cls_token_id]
          __UpperCamelCase											=  [self.question_token_id] + [self.convert_tokens_to_ids('''.'''			)]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep			) * [0]

          if self.padding_side == "right":
              # Input is question-then-context
              return len(cls + token_ids_a + question_suffix + sep			) * [0] + len(token_ids_a + sep			) * [1]
          else:
              # Input is context-then-question
              return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + question_suffix + sep			) * [1]







      def  snake_case		(       self   :	List[Any]	,    snake_case   :	str	,    snake_case   :	Optional[str] = None			):
          __UpperCamelCase											=  self._tokenizer.model.save(snake_case	,    name=snake_case			)
          return tuple(snake_case			)












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_										=       {
    "configuration_marian": ["MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP", "MarianConfig", "MarianOnnxConfig"],
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_										=       ["MarianTokenizer"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_										=       [
     "MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST",
     "MarianForCausalLM",
     "MarianModel",
     "MarianMTModel",
     "MarianPreTrainedModel",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_										=       ["TFMarianModel", "TFMarianMTModel", "TFMarianPreTrainedModel"]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_										=       ["FlaxMarianModel", "FlaxMarianMTModel", "FlaxMarianPreTrainedModel"]

if TYPE_CHECKING:
 from .configuration_marian import MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP, MarianConfig, MarianOnnxConfig

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_marian import MarianTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_marian import (
      MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST,
      MarianForCausalLM,
      MarianModel,
      MarianMTModel,
      MarianPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_marian import TFMarianModel, TFMarianMTModel, TFMarianPreTrainedModel

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_marian import FlaxMarianModel, FlaxMarianMTModel, FlaxMarianPreTrainedModel

else:
 import sys

 a_										=       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






