



# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
a_								=   abspath(join(dirname(dirname(dirname(__file__))), """src"""))
sys.path.insert(1, git_repo_path)


# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)





def 						a__     (						_UpperCamelCase     :	Optional[Any]							):
   from transformers.testing_utils import pytest_addoption_shared

   pytest_addoption_shared(_UpperCamelCase							)





def 						a__     (						_UpperCamelCase     :	int							):
   from transformers.testing_utils import pytest_terminal_summary_main

   __lowerCamelCase  							=      terminalreporter.config.getoption('''--make-reports'''							)
   if make_reports:
      pytest_terminal_summary_main(_UpperCamelCase	,id=_UpperCamelCase							)











import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


a_								=   logging.get_logger(__name__)

a_								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_								=   {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

a_								=   {
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}



class 	__lowerCAmelCase       (		lowerCAmelCase__				):
 lowerCAmelCase__       =					VOCAB_FILES_NAMES
 lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       =					["""input_ids""", """attention_mask"""]
 lowerCAmelCase__       =					LongformerTokenizer




 def __init__(  self		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase="replace"		,			__UpperCAmelCase="<s>"		,			__UpperCAmelCase="</s>"		,			__UpperCAmelCase="</s>"		,			__UpperCAmelCase="<s>"		,			__UpperCAmelCase="<unk>"		,			__UpperCAmelCase="<pad>"		,			__UpperCAmelCase="<mask>"		,			__UpperCAmelCase=False		,			__UpperCAmelCase=True		,			**__UpperCAmelCase		,			):



    '''simple docstring'''
    super().__init__(
        __UpperCAmelCase		,			__UpperCAmelCase		,			tokenizer_file=__UpperCAmelCase		,			errors=__UpperCAmelCase		,			bos_token=__UpperCAmelCase		,			eos_token=__UpperCAmelCase		,			sep_token=__UpperCAmelCase		,			cls_token=__UpperCAmelCase		,			unk_token=__UpperCAmelCase		,			pad_token=__UpperCAmelCase		,			mask_token=__UpperCAmelCase		,			add_prefix_space=__UpperCAmelCase		,			trim_offsets=__UpperCAmelCase		,			**__UpperCAmelCase		,			)

    __lowerCamelCase  							=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
    if pre_tok_state.get('''add_prefix_space'''		,			__UpperCAmelCase			) != add_prefix_space:
       __lowerCamelCase  							=      getattr(__UpperCAmelCase		,			pre_tok_state.pop('''type'''			)			)
       __lowerCamelCase  							=      add_prefix_space
       __lowerCamelCase  							=      pre_tok_class(**__UpperCAmelCase			)

    __lowerCamelCase  							=      add_prefix_space

    __lowerCamelCase  							=      '''post_processor'''
    __lowerCamelCase  							=      getattr(self.backend_tokenizer		,			__UpperCAmelCase		,			__UpperCAmelCase			)
    if tokenizer_component_instance:
       __lowerCamelCase  							=      json.loads(tokenizer_component_instance.__getstate__()			)

       # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
       if "sep" in state:
          __lowerCamelCase  							=      tuple(state['''sep''']			)
       if "cls" in state:
          __lowerCamelCase  							=      tuple(state['''cls''']			)

       __lowerCamelCase  							=      False

       if state.get('''add_prefix_space'''		,			__UpperCAmelCase			) != add_prefix_space:
          __lowerCamelCase  							=      add_prefix_space
          __lowerCamelCase  							=      True

       if state.get('''trim_offsets'''		,			__UpperCAmelCase			) != trim_offsets:
          __lowerCamelCase  							=      trim_offsets
          __lowerCamelCase  							=      True

       if changes_to_apply:
          __lowerCamelCase  							=      getattr(__UpperCAmelCase		,			state.pop('''type'''			)			)
          __lowerCamelCase  							=      component_class(**__UpperCAmelCase			)
          setattr(self.backend_tokenizer		,			__UpperCAmelCase		,			__UpperCAmelCase			)




 @property
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    if self._mask_token is None:
       if self.verbose:
          logger.error('''Using mask_token, but it is not set yet.'''			)
       return None
    return str(self._mask_token			)




 @mask_token.setter
 def     lowerCamelCase (  self		,			__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      AddedToken(__UpperCAmelCase		,			lstrip=__UpperCAmelCase		,			rstrip=__UpperCAmelCase			) if isinstance(__UpperCAmelCase		,			__UpperCAmelCase			) else value
    __lowerCamelCase  							=      value




 def     lowerCamelCase (  self		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      kwargs.get('''is_split_into_words'''		,			__UpperCAmelCase			)
    assert self.add_prefix_space or not is_split_into_words, (
        F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._batch_encode_plus(*__UpperCAmelCase		,			**__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      kwargs.get('''is_split_into_words'''		,			__UpperCAmelCase			)

    assert self.add_prefix_space or not is_split_into_words, (
        F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._encode_plus(*__UpperCAmelCase		,			**__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      self._tokenizer.model.save(__UpperCAmelCase		,			name=__UpperCAmelCase			)
    return tuple(__UpperCAmelCase			)




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase=None			):



    '''simple docstring'''
    __lowerCamelCase  							=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
    if token_ids_a is None:
       return output

    return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      [self.sep_token_id]
    __lowerCamelCase  							=      [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






