

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import ConvNextImageProcessor





class      snake_case_       (		unittest.TestCase ):







			'''simple docstring'''

			def __init__(					self			:    Any							,       _UpperCamelCase			:    Optional[Any]							,       _UpperCamelCase			:    str=7							,       _UpperCamelCase			:    int=3							,       _UpperCamelCase			:    Dict=1_8							,       _UpperCamelCase			:    List[str]=3_0							,       _UpperCamelCase			:    Any=4_0_0							,       _UpperCamelCase			:    int=True							,       _UpperCamelCase			:    Optional[int]=None							,       _UpperCamelCase			:    Union[str, Any]=0.875							,       _UpperCamelCase			:    Tuple=True							,       _UpperCamelCase			:    List[str]=[0.5, 0.5, 0.5]							,       _UpperCamelCase			:    int=[0.5, 0.5, 0.5]							,       )				->Tuple:
									snake_case_       =     size if size is not None else {'''shortest_edge''': 2_0}
									snake_case_       =     parent
									snake_case_       =     batch_size
									snake_case_       =     num_channels
									snake_case_       =     image_size
									snake_case_       =     min_resolution
									snake_case_       =     max_resolution
									snake_case_       =     do_resize
									snake_case_       =     size
									snake_case_       =     crop_pct
									snake_case_       =     do_normalize
									snake_case_       =     image_mean
									snake_case_       =     image_std







			def     snake_case__(					self			:    int							)				->Dict:
									return {
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_normalize": self.do_normalize,
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "crop_pct": self.crop_pct,
									}






@require_torch
@require_vision
class      snake_case_       (		__A     ,	unittest.TestCase ):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE					:       Optional[int]						      =    ConvNextImageProcessor if is_vision_available() else None







			def     snake_case__(					self			:    int							)				->Tuple:
									snake_case_       =     ConvNextImageProcessingTester(self							)







			@property
			def     snake_case__(					self			:    int							)				->Union[str, Any]:
									return self.image_processor_tester.prepare_image_processor_dict()







			def     snake_case__(					self			:    List[Any]							)				->int:
									snake_case_       =     self.image_processing_class(**self.image_processor_dict							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''do_resize'''							)							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''size'''							)							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''crop_pct'''							)							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''do_normalize'''							)							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''image_mean'''							)							)
									self.assertTrue(hasattr(_UpperCamelCase							,       '''image_std'''							)							)







			def     snake_case__(					self			:    Optional[Any]							)				->int:
									snake_case_       =     self.image_processing_class.from_dict(self.image_processor_dict							)
									self.assertEqual(image_processor.size							,       {'''shortest_edge''': 2_0}							)

									snake_case_       =     self.image_processing_class.from_dict(self.image_processor_dict							,       size=4_2							)
									self.assertEqual(image_processor.size							,       {'''shortest_edge''': 4_2}							)







			def     snake_case__(					self			:    Union[str, Any]							)				->List[Any]:
									pass







			def     snake_case__(					self			:    Any							)				->Union[str, Any]:
									# Initialize image_processing
									snake_case_       =     self.image_processing_class(**self.image_processor_dict							)
									# create random PIL images
									snake_case_       =     prepare_image_inputs(self.image_processor_tester							,       equal_resolution=_UpperCamelCase							)
									for image in image_inputs:
															self.assertIsInstance(_UpperCamelCase							,       Image.Image							)

									# Test not batched input
									snake_case_       =     image_processing(image_inputs[0]							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )

									# Test batched
									snake_case_       =     image_processing(_UpperCamelCase							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )







			def     snake_case__(					self			:    Optional[Any]							)				->Union[str, Any]:
									# Initialize image_processing
									snake_case_       =     self.image_processing_class(**self.image_processor_dict							)
									# create random numpy tensors
									snake_case_       =     prepare_image_inputs(self.image_processor_tester							,       equal_resolution=_UpperCamelCase							,       numpify=_UpperCamelCase							)
									for image in image_inputs:
															self.assertIsInstance(_UpperCamelCase							,       np.ndarray							)

									# Test not batched input
									snake_case_       =     image_processing(image_inputs[0]							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )

									# Test batched
									snake_case_       =     image_processing(_UpperCamelCase							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )







			def     snake_case__(					self			:    Tuple							)				->int:
									# Initialize image_processing
									snake_case_       =     self.image_processing_class(**self.image_processor_dict							)
									# create random PyTorch tensors
									snake_case_       =     prepare_image_inputs(self.image_processor_tester							,       equal_resolution=_UpperCamelCase							,       torchify=_UpperCamelCase							)
									for image in image_inputs:
															self.assertIsInstance(_UpperCamelCase							,       torch.Tensor							)

									# Test not batched input
									snake_case_       =     image_processing(image_inputs[0]							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )

									# Test batched
									snake_case_       =     image_processing(_UpperCamelCase							,       return_tensors='''pt'''							).pixel_values
									self.assertEqual(
									    encoded_images.shape							,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''shortest_edge'''],
									        self.image_processor_tester.size['''shortest_edge'''],
									    )							,       )


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase_	     =     logging.get_logger(__name__)

lowerCAmelCase_	     =     {'''vocab_file''': '''vocab.txt'''}

lowerCAmelCase_	     =     {
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

lowerCAmelCase_	     =     {
    '''tau/splinter-base''': 5_12,
    '''tau/splinter-base-qass''': 5_12,
    '''tau/splinter-large''': 5_12,
    '''tau/splinter-large-qass''': 5_12,
}

lowerCAmelCase_	     =     {
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}





class      snake_case_       (		__A ):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE					:       List[str]						      =    VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE					:       List[str]						      =    PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE					:       Optional[int]						      =    PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE					:       List[str]						      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE					:       Any						      =    SplinterTokenizer







			def __init__(					self			:    List[str]							,       _UpperCamelCase			:    Dict=None							,       _UpperCamelCase			:    Union[str, Any]=None							,       _UpperCamelCase			:    Union[str, Any]=True							,       _UpperCamelCase			:    List[Any]="[UNK]"							,       _UpperCamelCase			:    List[str]="[SEP]"							,       _UpperCamelCase			:    Optional[int]="[PAD]"							,       _UpperCamelCase			:    Tuple="[CLS]"							,       _UpperCamelCase			:    Optional[int]="[MASK]"							,       _UpperCamelCase			:    str="[QUESTION]"							,       _UpperCamelCase			:    Optional[Any]=True							,       _UpperCamelCase			:    Dict=None							,       **_UpperCamelCase			:    Optional[int]							,       )				->Union[str, Any]:
									super().__init__(
									    _UpperCamelCase							,       tokenizer_file=_UpperCamelCase							,       do_lower_case=_UpperCamelCase							,       unk_token=_UpperCamelCase							,       sep_token=_UpperCamelCase							,       pad_token=_UpperCamelCase							,       cls_token=_UpperCamelCase							,       mask_token=_UpperCamelCase							,       tokenize_chinese_chars=_UpperCamelCase							,       strip_accents=_UpperCamelCase							,       additional_special_tokens=(question_token,)							,       **_UpperCamelCase							,       )

									snake_case_       =     json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
									if (
									    pre_tok_state.get('''lowercase'''							,       _UpperCamelCase							) != do_lower_case
									    or pre_tok_state.get('''strip_accents'''							,       _UpperCamelCase							) != strip_accents
									):
															snake_case_       =     getattr(_UpperCamelCase							,       pre_tok_state.pop('''type'''							)							)
															snake_case_       =     do_lower_case
															snake_case_       =     strip_accents
															snake_case_       =     pre_tok_class(**_UpperCamelCase							)

									snake_case_       =     do_lower_case







			@property
			def     snake_case__(					self			:    Optional[int]							)				->int:
									return self.convert_tokens_to_ids(self.question_token							)







			def     snake_case__(					self			:    Dict							,       _UpperCamelCase			:    List[int]							,       _UpperCamelCase			:    Optional[List[int]] = None							)				->List[int]:
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									snake_case_       =     [self.cls_token_id]
									snake_case_       =     [self.sep_token_id]
									snake_case_       =     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''							)]
									if self.padding_side == "right":
															# Input is question-then-context
															return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
															# Input is context-then-question
															return cls + token_ids_a + sep + token_ids_a + question_suffix + sep







			def     snake_case__(					self			:    str							,       _UpperCamelCase			:    List[int]							,       _UpperCamelCase			:    Optional[List[int]] = None							)				->List[int]:
									snake_case_       =     [self.sep_token_id]
									snake_case_       =     [self.cls_token_id]
									snake_case_       =     [self.question_token_id] + [self.convert_tokens_to_ids('''.'''							)]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]

									if self.padding_side == "right":
															# Input is question-then-context
															return len(cls + token_ids_a + question_suffix + sep							) * [0] + len(token_ids_a + sep							) * [1]
									else:
															# Input is context-then-question
															return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + question_suffix + sep							) * [1]







			def     snake_case__(					self			:    Union[str, Any]							,       _UpperCamelCase			:    str							,       _UpperCamelCase			:    Optional[str] = None							)				->Tuple[str]:
									snake_case_       =     self._tokenizer.model.save(_UpperCamelCase							,       name=_UpperCamelCase							)
									return tuple(_UpperCamelCase							)