





'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__UpperCAmelCase         =   logging.get_logger(__name__)

__UpperCAmelCase         =   {"""vocab_file""": """vocab.txt"""}

__UpperCAmelCase         =   {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

__UpperCAmelCase         =   {
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

__UpperCAmelCase         =   {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}




class       UpperCamelCase__			(			lowercase_	):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__						=					VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE__						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE__						=					SplinterTokenizer


					def __init__(		self							:	int     ,		lowerCamelCase_							:	str=None     ,		lowerCamelCase_							:	Tuple=None     ,		lowerCamelCase_							:	Tuple=True     ,		lowerCamelCase_							:	str="[UNK]"     ,		lowerCamelCase_							:	int="[SEP]"     ,		lowerCamelCase_							:	List[str]="[PAD]"     ,		lowerCamelCase_							:	List[Any]="[CLS]"     ,		lowerCamelCase_							:	Optional[Any]="[MASK]"     ,		lowerCamelCase_							:	List[str]="[QUESTION]"     ,		lowerCamelCase_							:	str=True     ,		lowerCamelCase_							:	Tuple=None     ,		**lowerCamelCase_							:	Tuple     ,		):

							'''simple docstring'''



							super().__init__(
							    lowerCamelCase_     ,		tokenizer_file=lowerCamelCase_     ,		do_lower_case=lowerCamelCase_     ,		unk_token=lowerCamelCase_     ,		sep_token=lowerCamelCase_     ,		pad_token=lowerCamelCase_     ,		cls_token=lowerCamelCase_     ,		mask_token=lowerCamelCase_     ,		tokenize_chinese_chars=lowerCamelCase_     ,		strip_accents=lowerCamelCase_     ,		additional_special_tokens=(question_token,)     ,		**lowerCamelCase_     ,		)

							SCREAMING_SNAKE_CASE				:							str						=	json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
							if (
							    pre_tok_state.get("""lowercase"""     ,		lowerCamelCase_		) != do_lower_case
							    or pre_tok_state.get("""strip_accents"""     ,		lowerCamelCase_		) != strip_accents
							):
									SCREAMING_SNAKE_CASE				:							Optional[int]						=	getattr(lowerCamelCase_     ,		pre_tok_state.pop("""type"""		)		)
									SCREAMING_SNAKE_CASE				:							int						=	do_lower_case
									SCREAMING_SNAKE_CASE				:							int						=	strip_accents
									SCREAMING_SNAKE_CASE				:							Dict						=	pre_tok_class(**lowerCamelCase_		)

							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	do_lower_case


					@property
					def lowerCamelCase_	(		self							:	Any		):

							'''simple docstring'''



							return self.convert_tokens_to_ids(self.question_token		)


					def lowerCamelCase_	(		self							:	List[Any]     ,		lowerCamelCase_							:	List[int]     ,		lowerCamelCase_							:	Optional[List[int]] = None		):

							'''simple docstring'''



							if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							SCREAMING_SNAKE_CASE				:							Optional[int]						=	[self.cls_token_id]
							SCREAMING_SNAKE_CASE				:							List[str]						=	[self.sep_token_id]
							SCREAMING_SNAKE_CASE				:							int						=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""		)]
							if self.padding_side == "right":
									# Input is question-then-context
									return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
							else:
									# Input is context-then-question
									return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


					def lowerCamelCase_	(		self							:	int     ,		lowerCamelCase_							:	List[int]     ,		lowerCamelCase_							:	Optional[List[int]] = None		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	[self.sep_token_id]
							SCREAMING_SNAKE_CASE				:							List[str]						=	[self.cls_token_id]
							SCREAMING_SNAKE_CASE				:							Tuple						=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""		)]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep		) * [0]

							if self.padding_side == "right":
									# Input is question-then-context
									return len(cls + token_ids_a + question_suffix + sep		) * [0] + len(token_ids_a + sep		) * [1]
							else:
									# Input is context-then-question
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + question_suffix + sep		) * [1]



					def lowerCamelCase_	(		self							:	Optional[int]     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	Optional[str] = None		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self._tokenizer.model.save(lowerCamelCase_     ,		name=lowerCamelCase_		)
							return tuple(lowerCamelCase_		)










'''simple docstring'''




import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
							from PIL import Image




class       UpperCamelCase__			(			unittest.TestCase		,			lowercase_	):




					"""simple docstring"""


					def lowerCamelCase_	(		self							:	Any		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							int						=	load_tool("""image-captioning"""		)
							self.tool.setup()
							SCREAMING_SNAKE_CASE				:							str						=	load_tool("""image-captioning"""     ,		remote=lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	List[Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""		)		) / """000000039769.png"""		)
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self.tool(lowerCamelCase_		)
							self.assertEqual(lowerCamelCase_     ,		"""two cats sleeping on a couch"""		)


					def lowerCamelCase_	(		self							:	str		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""		)		) / """000000039769.png"""		)
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	self.remote_tool(lowerCamelCase_		)
							self.assertEqual(lowerCamelCase_     ,		"""two cats sleeping on a couch"""		)


					def lowerCamelCase_	(		self							:	List[Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""		)		) / """000000039769.png"""		)
							SCREAMING_SNAKE_CASE				:							str						=	self.tool(image=lowerCamelCase_		)
							self.assertEqual(lowerCamelCase_     ,		"""two cats sleeping on a couch"""		)



					def lowerCamelCase_	(		self							:	Any		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""		)		) / """000000039769.png"""		)
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	self.remote_tool(image=lowerCamelCase_		)
							self.assertEqual(lowerCamelCase_     ,		"""two cats sleeping on a couch"""		)



