



import json
import os
import tempfile

from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
  import numpy as np
  import torch

if is_vision_available():
  from PIL import Image





def 						a__     (						_UpperCamelCase     :	str	,_UpperCamelCase     :	int=False	,_UpperCamelCase     :	Tuple=False	,_UpperCamelCase     :	int=False							):
   assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

   __lowerCamelCase  							=      []
   for i in range(image_processor_tester.batch_size							):
      if equal_resolution:
         __lowerCamelCase  							=      __lowerCamelCase  							=      image_processor_tester.max_resolution
      else:
         # To avoid getting image width/height 0
         __lowerCamelCase  							=      image_processor_tester.min_resolution
         if getattr(_UpperCamelCase	,'''size_divisor'''	,_UpperCamelCase							):
            # If `size_divisor` is defined, the image needs to have width/size >= `size_divisor`
            __lowerCamelCase  							=      max(image_processor_tester.size_divisor	,_UpperCamelCase							)
         __lowerCamelCase							,__lowerCamelCase  							=      np.random.choice(np.arange(_UpperCamelCase	,image_processor_tester.max_resolution							)	,2							)
      image_inputs.append(
          np.random.randint(2_55	,size=(image_processor_tester.num_channels, width, height)	,dtype=np.uinta							)							)

   if not numpify and not torchify:
      # PIL expects the channel dimension as last dimension
      __lowerCamelCase  							=      [Image.fromarray(np.moveaxis(_UpperCamelCase	,0	,-1							)							) for image in image_inputs]

   if torchify:
      __lowerCamelCase  							=      [torch.from_numpy(_UpperCamelCase							) for image in image_inputs]

   return image_inputs





def 						a__     (						_UpperCamelCase     :	int	,_UpperCamelCase     :	Optional[Any]=10	,_UpperCamelCase     :	int=10	,_UpperCamelCase     :	Dict=False	,_UpperCamelCase     :	List[str]=False							):
   __lowerCamelCase  							=      []
   for i in range(image_processor_tester.num_frames							):
      video.append(np.random.randint(2_55	,size=(image_processor_tester.num_channels, width, height)	,dtype=np.uinta							)							)

   if not numpify and not torchify:
      # PIL expects the channel dimension as last dimension
      __lowerCamelCase  							=      [Image.fromarray(np.moveaxis(_UpperCamelCase	,0	,-1							)							) for frame in video]

   if torchify:
      __lowerCamelCase  							=      [torch.from_numpy(_UpperCamelCase							) for frame in video]

   return video





def 						a__     (						_UpperCamelCase     :	Tuple	,_UpperCamelCase     :	Dict=False	,_UpperCamelCase     :	Optional[int]=False	,_UpperCamelCase     :	Optional[Any]=False							):
   assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

   __lowerCamelCase  							=      []
   for i in range(image_processor_tester.batch_size							):
      if equal_resolution:
         __lowerCamelCase  							=      __lowerCamelCase  							=      image_processor_tester.max_resolution
      else:
         __lowerCamelCase							,__lowerCamelCase  							=      np.random.choice(
             np.arange(image_processor_tester.min_resolution	,image_processor_tester.max_resolution							)	,2							)
         __lowerCamelCase  							=      prepare_video(
             image_processor_tester=_UpperCamelCase	,width=_UpperCamelCase	,height=_UpperCamelCase	,numpify=_UpperCamelCase	,torchify=_UpperCamelCase	,)
      video_inputs.append(_UpperCamelCase							)

   return video_inputs





class 	__lowerCAmelCase       :
 lowerCAmelCase__       =					None




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      self.image_processing_class(**self.image_processor_dict			)
    __lowerCamelCase  							=      json.loads(image_processor.to_json_string()			)
    for key, value in self.image_processor_dict.items():
       self.assertEqual(obj[key]		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      self.image_processing_class(**self.image_processor_dict			)

    with tempfile.TemporaryDirectory() as tmpdirname:
       __lowerCamelCase  							=      os.path.join(__UpperCAmelCase		,			'''image_processor.json'''			)
       image_processor_first.to_json_file(__UpperCAmelCase			)
       __lowerCamelCase  							=      self.image_processing_class.from_json_file(__UpperCAmelCase			)

    self.assertEqual(image_processor_second.to_dict()		,			image_processor_first.to_dict()			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      self.image_processing_class(**self.image_processor_dict			)

    with tempfile.TemporaryDirectory() as tmpdirname:
       __lowerCamelCase  							=      image_processor_first.save_pretrained(__UpperCAmelCase			)[0]
       check_json_file_has_correct_format(__UpperCAmelCase			)
       __lowerCamelCase  							=      self.image_processing_class.from_pretrained(__UpperCAmelCase			)

    self.assertEqual(image_processor_second.to_dict()		,			image_processor_first.to_dict()			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      self.image_processing_class()
    self.assertIsNotNone(__UpperCAmelCase			)




 @require_torch
 @require_vision
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    if self.test_cast_dtype is not None:
       # Initialize image_processor
       __lowerCamelCase  							=      self.image_processing_class(**self.image_processor_dict			)

       # create random PyTorch tensors
       __lowerCamelCase  							=      prepare_image_inputs(self.image_processor_tester		,			equal_resolution=__UpperCAmelCase		,			torchify=__UpperCAmelCase			)

       __lowerCamelCase  							=      image_processor(__UpperCAmelCase		,			return_tensors='''pt'''			)
       # for layoutLM compatiblity
       self.assertEqual(encoding.pixel_values.device		,			torch.device('''cpu'''			)			)
       self.assertEqual(encoding.pixel_values.dtype		,			torch.floataa			)

       __lowerCamelCase  							=      image_processor(__UpperCAmelCase		,			return_tensors='''pt'''			).to(torch.floataa			)
       self.assertEqual(encoding.pixel_values.device		,			torch.device('''cpu'''			)			)
       self.assertEqual(encoding.pixel_values.dtype		,			torch.floataa			)

       __lowerCamelCase  							=      image_processor(__UpperCAmelCase		,			return_tensors='''pt'''			).to('''cpu'''		,			torch.bfloataa			)
       self.assertEqual(encoding.pixel_values.device		,			torch.device('''cpu'''			)			)
       self.assertEqual(encoding.pixel_values.dtype		,			torch.bfloataa			)

       with self.assertRaises(__UpperCAmelCase			):
          __lowerCamelCase  							=      image_processor(__UpperCAmelCase		,			return_tensors='''pt'''			).to(torch.bfloataa		,			'''cpu'''			)

       # Try with text + image feature
       __lowerCamelCase  							=      image_processor(__UpperCAmelCase		,			return_tensors='''pt'''			)
       encoding.update({'''input_ids''': torch.LongTensor([[1, 2, 3], [4, 5, 6]]			)}			)
       __lowerCamelCase  							=      encoding.to(torch.floataa			)

       self.assertEqual(encoding.pixel_values.device		,			torch.device('''cpu'''			)			)
       self.assertEqual(encoding.pixel_values.dtype		,			torch.floataa			)
       self.assertEqual(encoding.input_ids.dtype		,			torch.long			)











import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


a_								=   logging.get_logger(__name__)

a_								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_								=   {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

a_								=   {
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}



class 	__lowerCAmelCase       (		lowerCAmelCase__				):
 lowerCAmelCase__       =					VOCAB_FILES_NAMES
 lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       =					["""input_ids""", """attention_mask"""]
 lowerCAmelCase__       =					LongformerTokenizer




 def __init__(  self		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase="replace"		,			__UpperCAmelCase="<s>"		,			__UpperCAmelCase="</s>"		,			__UpperCAmelCase="</s>"		,			__UpperCAmelCase="<s>"		,			__UpperCAmelCase="<unk>"		,			__UpperCAmelCase="<pad>"		,			__UpperCAmelCase="<mask>"		,			__UpperCAmelCase=False		,			__UpperCAmelCase=True		,			**__UpperCAmelCase		,			):



    '''simple docstring'''
    super().__init__(
        __UpperCAmelCase		,			__UpperCAmelCase		,			tokenizer_file=__UpperCAmelCase		,			errors=__UpperCAmelCase		,			bos_token=__UpperCAmelCase		,			eos_token=__UpperCAmelCase		,			sep_token=__UpperCAmelCase		,			cls_token=__UpperCAmelCase		,			unk_token=__UpperCAmelCase		,			pad_token=__UpperCAmelCase		,			mask_token=__UpperCAmelCase		,			add_prefix_space=__UpperCAmelCase		,			trim_offsets=__UpperCAmelCase		,			**__UpperCAmelCase		,			)

    __lowerCamelCase  							=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
    if pre_tok_state.get('''add_prefix_space'''		,			__UpperCAmelCase			) != add_prefix_space:
       __lowerCamelCase  							=      getattr(__UpperCAmelCase		,			pre_tok_state.pop('''type'''			)			)
       __lowerCamelCase  							=      add_prefix_space
       __lowerCamelCase  							=      pre_tok_class(**__UpperCAmelCase			)

    __lowerCamelCase  							=      add_prefix_space

    __lowerCamelCase  							=      '''post_processor'''
    __lowerCamelCase  							=      getattr(self.backend_tokenizer		,			__UpperCAmelCase		,			__UpperCAmelCase			)
    if tokenizer_component_instance:
       __lowerCamelCase  							=      json.loads(tokenizer_component_instance.__getstate__()			)

       # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
       if "sep" in state:
          __lowerCamelCase  							=      tuple(state['''sep''']			)
       if "cls" in state:
          __lowerCamelCase  							=      tuple(state['''cls''']			)

       __lowerCamelCase  							=      False

       if state.get('''add_prefix_space'''		,			__UpperCAmelCase			) != add_prefix_space:
          __lowerCamelCase  							=      add_prefix_space
          __lowerCamelCase  							=      True

       if state.get('''trim_offsets'''		,			__UpperCAmelCase			) != trim_offsets:
          __lowerCamelCase  							=      trim_offsets
          __lowerCamelCase  							=      True

       if changes_to_apply:
          __lowerCamelCase  							=      getattr(__UpperCAmelCase		,			state.pop('''type'''			)			)
          __lowerCamelCase  							=      component_class(**__UpperCAmelCase			)
          setattr(self.backend_tokenizer		,			__UpperCAmelCase		,			__UpperCAmelCase			)




 @property
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    if self._mask_token is None:
       if self.verbose:
          logger.error('''Using mask_token, but it is not set yet.'''			)
       return None
    return str(self._mask_token			)




 @mask_token.setter
 def     lowerCamelCase (  self		,			__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      AddedToken(__UpperCAmelCase		,			lstrip=__UpperCAmelCase		,			rstrip=__UpperCAmelCase			) if isinstance(__UpperCAmelCase		,			__UpperCAmelCase			) else value
    __lowerCamelCase  							=      value




 def     lowerCamelCase (  self		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      kwargs.get('''is_split_into_words'''		,			__UpperCAmelCase			)
    assert self.add_prefix_space or not is_split_into_words, (
        F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._batch_encode_plus(*__UpperCAmelCase		,			**__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      kwargs.get('''is_split_into_words'''		,			__UpperCAmelCase			)

    assert self.add_prefix_space or not is_split_into_words, (
        F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._encode_plus(*__UpperCAmelCase		,			**__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      self._tokenizer.model.save(__UpperCAmelCase		,			name=__UpperCAmelCase			)
    return tuple(__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase=None			):



    '''simple docstring'''
    __lowerCamelCase  							=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
    if token_ids_a is None:
       return output

    return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      [self.sep_token_id]
    __lowerCamelCase  							=      [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






