

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 			__lowerCAmelCase					( SCREAMING_SNAKE_CASE      ,      unittest.TestCase							):
  _a						      =  RealmTokenizer
  _a						      =  RealmTokenizerFast
  _a						      =  True
  _a						      =  True
  _a						      =  filter_non_english




  def   A__					(    self )  ->			Any:
        '''simple docstring'''






        super().setUp()

        _lowercase     	=[
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        _lowercase     	=os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['vocab_file'] )
        with open(self.vocab_file							,       'w'							,       encoding='utf-8' ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )




  def   A__					(    self							,       lowerCAmelCase )  ->			Dict:
        '''simple docstring'''






        _lowercase     	='UNwant\u00E9d,running'
        _lowercase     	='unwanted, running'
        return input_text, output_text




  def   A__					(    self )  ->			Any:
        '''simple docstring'''






        _lowercase     	=self.tokenizer_class(self.vocab_file )

        _lowercase     	=tokenizer.tokenize('UNwant\u00E9d,running' )
        self.assertListEqual(lowerCAmelCase							,       ['un', '##want', '##ed', ',', 'runn', '##ing'] )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase )							,       [9, 6, 7, 12, 10, 11] )




  def   A__					(    self )  ->			str:
        '''simple docstring'''






        if not self.test_rust_tokenizer:
              return

        _lowercase     	=self.get_tokenizer()
        _lowercase     	=self.get_rust_tokenizer()

        _lowercase     	='UNwant\u00E9d,running'

        _lowercase     	=tokenizer.tokenize(lowerCAmelCase )
        _lowercase     	=rust_tokenizer.tokenize(lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )

        _lowercase     	=tokenizer.encode(lowerCAmelCase							,       add_special_tokens=lowerCAmelCase )
        _lowercase     	=rust_tokenizer.encode(lowerCAmelCase							,       add_special_tokens=lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )

        _lowercase     	=self.get_rust_tokenizer()
        _lowercase     	=tokenizer.encode(lowerCAmelCase )
        _lowercase     	=rust_tokenizer.encode(lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )

        # With lower casing
        _lowercase     	=self.get_tokenizer(do_lower_case=lowerCAmelCase )
        _lowercase     	=self.get_rust_tokenizer(do_lower_case=lowerCAmelCase )

        _lowercase     	='UNwant\u00E9d,running'

        _lowercase     	=tokenizer.tokenize(lowerCAmelCase )
        _lowercase     	=rust_tokenizer.tokenize(lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )

        _lowercase     	=tokenizer.encode(lowerCAmelCase							,       add_special_tokens=lowerCAmelCase )
        _lowercase     	=rust_tokenizer.encode(lowerCAmelCase							,       add_special_tokens=lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )

        _lowercase     	=self.get_rust_tokenizer()
        _lowercase     	=tokenizer.encode(lowerCAmelCase )
        _lowercase     	=rust_tokenizer.encode(lowerCAmelCase )
        self.assertListEqual(lowerCAmelCase							,       lowerCAmelCase )




  def   A__					(    self )  ->			Tuple:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )							,       ['ah', '\u535A', '\u63A8', 'zz'] )




  def   A__					(    self )  ->			Dict:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )							,       ['hello', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )							,       ['hello'] )




  def   A__					(    self )  ->			Optional[int]:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase							,       strip_accents=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )							,       ['hällo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )							,       ['h\u00E9llo'] )




  def   A__					(    self )  ->			Any:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase							,       strip_accents=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )							,       ['hallo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )							,       ['hello'] )




  def   A__					(    self )  ->			int:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )							,       ['hallo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )							,       ['hello'] )




  def   A__					(    self )  ->			Any:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )							,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )




  def   A__					(    self )  ->			Optional[Any]:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase							,       strip_accents=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )							,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )




  def   A__					(    self )  ->			Any:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase							,       strip_accents=lowerCAmelCase )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )							,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )




  def   A__					(    self )  ->			Union[str, Any]:
        '''simple docstring'''






        _lowercase     	=BasicTokenizer(do_lower_case=lowerCAmelCase							,       never_split=['[UNK]'] )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )							,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )




  def   A__					(    self )  ->			List[str]:
        '''simple docstring'''






        _lowercase     	=['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

        _lowercase     	={}
        for i, token in enumerate(lowerCAmelCase ):
              _lowercase     	=i
        _lowercase     	=WordpieceTokenizer(vocab=lowerCAmelCase							,       unk_token='[UNK]' )

        self.assertListEqual(tokenizer.tokenize('' )							,       [] )

        self.assertListEqual(tokenizer.tokenize('unwanted running' )							,       ['un', '##want', '##ed', 'runn', '##ing'] )

        self.assertListEqual(tokenizer.tokenize('unwantedX running' )							,       ['[UNK]', 'runn', '##ing'] )




  def   A__					(    self )  ->			Optional[int]:
        '''simple docstring'''






        self.assertTrue(_is_whitespace(' ' ) )
        self.assertTrue(_is_whitespace('\t' ) )
        self.assertTrue(_is_whitespace('\r' ) )
        self.assertTrue(_is_whitespace('\n' ) )
        self.assertTrue(_is_whitespace('\u00A0' ) )

        self.assertFalse(_is_whitespace('A' ) )
        self.assertFalse(_is_whitespace('-' ) )




  def   A__					(    self )  ->			List[Any]:
        '''simple docstring'''






        self.assertTrue(_is_control('\u0005' ) )

        self.assertFalse(_is_control('A' ) )
        self.assertFalse(_is_control(' ' ) )
        self.assertFalse(_is_control('\t' ) )
        self.assertFalse(_is_control('\r' ) )




  def   A__					(    self )  ->			Union[str, Any]:
        '''simple docstring'''






        self.assertTrue(_is_punctuation('-' ) )
        self.assertTrue(_is_punctuation('$' ) )
        self.assertTrue(_is_punctuation('`' ) )
        self.assertTrue(_is_punctuation('.' ) )

        self.assertFalse(_is_punctuation('A' ) )
        self.assertFalse(_is_punctuation(' ' ) )




  def   A__					(    self )  ->			List[Any]:
        '''simple docstring'''






        _lowercase     	=self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(lowerCAmelCase ) for t in ['Test', '\xad', 'test']]							,       [['[UNK]'], [], ['[UNK]']] )

        if self.test_rust_tokenizer:
              _lowercase     	=self.get_rust_tokenizer()
              self.assertListEqual(
                  [rust_tokenizer.tokenize(lowerCAmelCase ) for t in ['Test', '\xad', 'test']]							,       [['[UNK]'], [], ['[UNK]']] )




  @slow
  def   A__					(    self )  ->			Optional[int]:
        '''simple docstring'''






        _lowercase     	=self.tokenizer_class.from_pretrained('bert-base-uncased' )

        _lowercase     	=tokenizer.encode('sequence builders'							,       add_special_tokens=lowerCAmelCase )
        _lowercase     	=tokenizer.encode('multi-sequence build'							,       add_special_tokens=lowerCAmelCase )

        _lowercase     	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase )
        _lowercase     	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase							,       lowerCAmelCase )

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]




  def   A__					(    self )  ->			List[Any]:
        '''simple docstring'''






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                    _lowercase     	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase							,       **lowerCAmelCase )

                    _lowercase     	=F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                    _lowercase     	=tokenizer_r.encode_plus(
                        lowerCAmelCase							,       return_attention_mask=lowerCAmelCase							,       return_token_type_ids=lowerCAmelCase							,       return_offsets_mapping=lowerCAmelCase							,       add_special_tokens=lowerCAmelCase							,       )

                    _lowercase     	=tokenizer_r.do_lower_case if hasattr(lowerCAmelCase							,       'do_lower_case' ) else False
                    _lowercase     	=(
                        [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), 'A'),
                            ((1, 2), ','),
                            ((3, 5), 'na'),
                            ((5, 6), '##ï'),
                            ((6, 8), '##ve'),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), 'Allen'),
                            ((21, 23), '##NL'),
                            ((23, 24), '##P'),
                            ((25, 33), 'sentence'),
                            ((33, 34), '.'),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                        if not do_lower_case
                        else [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), 'a'),
                            ((1, 2), ','),
                            ((3, 8), 'naive'),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), 'allen'),
                            ((21, 23), '##nl'),
                            ((23, 24), '##p'),
                            ((25, 33), 'sentence'),
                            ((33, 34), '.'),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                    )

                    self.assertEqual(
                        [e[1] for e in expected_results]							,       tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
                    self.assertEqual([e[0] for e in expected_results]							,       tokens['offset_mapping'] )




  @slow
  def   A__					(    self )  ->			Union[str, Any]:
        '''simple docstring'''






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                    _lowercase     	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase							,       **lowerCAmelCase )
                    _lowercase     	=self.tokenizer_class.from_pretrained(lowerCAmelCase							,       **lowerCAmelCase )
                    _lowercase     	=[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                    _lowercase     	=tokenizer_r.batch_encode_candidates(lowerCAmelCase							,       max_length=10							,       return_tensors='np' )
                    _lowercase     	=tokenizer_p.batch_encode_candidates(lowerCAmelCase							,       max_length=10							,       return_tensors='np' )

                    _lowercase     	=(2, 2, 10)

                    self.assertEqual(encoded_sentence_r['input_ids'].shape							,       lowerCAmelCase )
                    self.assertEqual(encoded_sentence_r['attention_mask'].shape							,       lowerCAmelCase )
                    self.assertEqual(encoded_sentence_r['token_type_ids'].shape							,       lowerCAmelCase )

                    self.assertEqual(encoded_sentence_p['input_ids'].shape							,       lowerCAmelCase )
                    self.assertEqual(encoded_sentence_p['attention_mask'].shape							,       lowerCAmelCase )
                    self.assertEqual(encoded_sentence_p['token_type_ids'].shape							,       lowerCAmelCase )





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase_     =       {
    'configuration_oneformer': ['ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'OneFormerConfig'],
    'processing_oneformer': ['OneFormerProcessor'],
}

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_     =       ['OneFormerImageProcessor']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_     =       [
      'ONEFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
      'OneFormerForUniversalSegmentation',
      'OneFormerModel',
      'OneFormerPreTrainedModel',
  ]

if TYPE_CHECKING:
  from .configuration_oneformer import ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, OneFormerConfig
  from .processing_oneformer import OneFormerProcessor

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .image_processing_oneformer import OneFormerImageProcessor
  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_oneformer import (
        ONEFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        OneFormerForUniversalSegmentation,
        OneFormerModel,
        OneFormerPreTrainedModel,
    )


else:
  import sys

  lowercase_     =       _LazyModule(__name__, globals()['__file__'], _import_structure)


