


'''simple docstring'''

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 					snake_case   (   lowercase      ,						unittest.TestCase      ):





  """simple docstring"""




  _lowerCamelCase		   =  RealmTokenizer
  _lowerCamelCase		   =  RealmTokenizerFast
  _lowerCamelCase		   =  True
  _lowerCamelCase		   =  True
  _lowerCamelCase		   =  filter_non_english






  def   snake_case		(   self					):







   """simple docstring"""
   super().setUp()

   lowerCamelCase_								=  [
       "[UNK]",
       "[CLS]",
       "[SEP]",
       "[PAD]",
       "[MASK]",
       "want",
       "##want",
       "##ed",
       "wa",
       "un",
       "runn",
       "##ing",
       ",",
       "low",
       "lowest",
   ]
   lowerCamelCase_								=  os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES["vocab_file"]					)
   with open(self.vocab_file      ,  "w"      ,  encoding="utf-8"					) as vocab_writer:
    vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)






  def   snake_case		(   self      ,  UpperCamelCase					):







   """simple docstring"""
   lowerCamelCase_								=  "UNwant\u00E9d,running"
   lowerCamelCase_								=  "unwanted, running"
   return input_text, output_text






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  self.tokenizer_class(self.vocab_file					)

   lowerCamelCase_								=  tokenizer.tokenize("UNwant\u00E9d,running"					)
   self.assertListEqual(UpperCamelCase      ,  ["un", "##want", "##ed", ",", "runn", "##ing"]					)
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase					)      ,  [9, 6, 7, 12, 10, 11]					)






  def   snake_case		(   self					):







   """simple docstring"""
   if not self.test_rust_tokenizer:
    return

   lowerCamelCase_								=  self.get_tokenizer()
   lowerCamelCase_								=  self.get_rust_tokenizer()

   lowerCamelCase_								=  "UNwant\u00E9d,running"

   lowerCamelCase_								=  tokenizer.tokenize(UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.tokenize(UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)

   lowerCamelCase_								=  tokenizer.encode(UpperCamelCase      ,  add_special_tokens=UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.encode(UpperCamelCase      ,  add_special_tokens=UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)

   lowerCamelCase_								=  self.get_rust_tokenizer()
   lowerCamelCase_								=  tokenizer.encode(UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.encode(UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)

   # With lower casing
   lowerCamelCase_								=  self.get_tokenizer(do_lower_case=UpperCamelCase					)
   lowerCamelCase_								=  self.get_rust_tokenizer(do_lower_case=UpperCamelCase					)

   lowerCamelCase_								=  "UNwant\u00E9d,running"

   lowerCamelCase_								=  tokenizer.tokenize(UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.tokenize(UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)

   lowerCamelCase_								=  tokenizer.encode(UpperCamelCase      ,  add_special_tokens=UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.encode(UpperCamelCase      ,  add_special_tokens=UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)

   lowerCamelCase_								=  self.get_rust_tokenizer()
   lowerCamelCase_								=  tokenizer.encode(UpperCamelCase					)
   lowerCamelCase_								=  rust_tokenizer.encode(UpperCamelCase					)
   self.assertListEqual(UpperCamelCase      ,  UpperCamelCase					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)      ,  ["ah", "\u535A", "\u63A8", "zz"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)      ,  ["hello", "!", "how", "are", "you", "?"]					)
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,  ["hello"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase      ,  strip_accents=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,  ["hällo", "!", "how", "are", "you", "?"]					)
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,  ["h\u00E9llo"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase      ,  strip_accents=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,  ["hallo", "!", "how", "are", "you", "?"]					)
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,  ["hello"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,  ["hallo", "!", "how", "are", "you", "?"]					)
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,  ["hello"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)      ,  ["HeLLo", "!", "how", "Are", "yoU", "?"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase      ,  strip_accents=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,  ["HäLLo", "!", "how", "Are", "yoU", "?"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase      ,  strip_accents=UpperCamelCase					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,  ["HaLLo", "!", "how", "Are", "yoU", "?"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  BasicTokenizer(do_lower_case=UpperCamelCase      ,  never_split=["[UNK]"]					)

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)      ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

   lowerCamelCase_								=  {}
   for i, token in enumerate(UpperCamelCase					):
    lowerCamelCase_								=  i
   lowerCamelCase_								=  WordpieceTokenizer(vocab=UpperCamelCase      ,  unk_token="[UNK]"					)

   self.assertListEqual(tokenizer.tokenize(""					)      ,  []					)

   self.assertListEqual(tokenizer.tokenize("unwanted running"					)      ,  ["un", "##want", "##ed", "runn", "##ing"]					)

   self.assertListEqual(tokenizer.tokenize("unwantedX running"					)      ,  ["[UNK]", "runn", "##ing"]					)






  def   snake_case		(   self					):







   """simple docstring"""
   self.assertTrue(_is_whitespace(" "					)					)
   self.assertTrue(_is_whitespace("\t"					)					)
   self.assertTrue(_is_whitespace("\r"					)					)
   self.assertTrue(_is_whitespace("\n"					)					)
   self.assertTrue(_is_whitespace("\u00A0"					)					)

   self.assertFalse(_is_whitespace("A"					)					)
   self.assertFalse(_is_whitespace("-"					)					)






  def   snake_case		(   self					):







   """simple docstring"""
   self.assertTrue(_is_control("\u0005"					)					)

   self.assertFalse(_is_control("A"					)					)
   self.assertFalse(_is_control(" "					)					)
   self.assertFalse(_is_control("\t"					)					)
   self.assertFalse(_is_control("\r"					)					)






  def   snake_case		(   self					):







   """simple docstring"""
   self.assertTrue(_is_punctuation("-"					)					)
   self.assertTrue(_is_punctuation("$"					)					)
   self.assertTrue(_is_punctuation("`"					)					)
   self.assertTrue(_is_punctuation("."					)					)

   self.assertFalse(_is_punctuation("A"					)					)
   self.assertFalse(_is_punctuation(" "					)					)






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(UpperCamelCase					) for t in ["Test", "\xad", "test"]]      ,  [["[UNK]"], [], ["[UNK]"]]					)

   if self.test_rust_tokenizer:
    lowerCamelCase_								=  self.get_rust_tokenizer()
    self.assertListEqual(
        [rust_tokenizer.tokenize(UpperCamelCase					) for t in ["Test", "\xad", "test"]]      ,  [["[UNK]"], [], ["[UNK]"]]					)






  @slow
  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  self.tokenizer_class.from_pretrained("bert-base-uncased"					)

   lowerCamelCase_								=  tokenizer.encode("sequence builders"      ,  add_special_tokens=UpperCamelCase					)
   lowerCamelCase_								=  tokenizer.encode("multi-sequence build"      ,  add_special_tokens=UpperCamelCase					)

   lowerCamelCase_								=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase					)
   lowerCamelCase_								=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase      ,  UpperCamelCase					)

   assert encoded_sentence == [101] + text + [102]
   assert encoded_pair == [101] + text + [102] + text_a + [102]






  def   snake_case		(   self					):







   """simple docstring"""
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
     lowerCamelCase_								=  self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,  **UpperCamelCase					)

     lowerCamelCase_								=  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
     lowerCamelCase_								=  tokenizer_r.encode_plus(
         UpperCamelCase      ,  return_attention_mask=UpperCamelCase      ,  return_token_type_ids=UpperCamelCase      ,  return_offsets_mapping=UpperCamelCase      ,  add_special_tokens=UpperCamelCase      ,  )

     lowerCamelCase_								=  tokenizer_r.do_lower_case if hasattr(UpperCamelCase      ,  "do_lower_case"					) else False
     lowerCamelCase_								=  (
         [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), "A"),
             ((1, 2), ","),
             ((3, 5), "na"),
             ((5, 6), "##ï"),
             ((6, 8), "##ve"),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), "Allen"),
             ((21, 23), "##NL"),
             ((23, 24), "##P"),
             ((25, 33), "sentence"),
             ((33, 34), "."),
             ((0, 0), tokenizer_r.sep_token),
         ]
         if not do_lower_case
         else [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), "a"),
             ((1, 2), ","),
             ((3, 8), "naive"),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), "allen"),
             ((21, 23), "##nl"),
             ((23, 24), "##p"),
             ((25, 33), "sentence"),
             ((33, 34), "."),
             ((0, 0), tokenizer_r.sep_token),
         ]
     )

     self.assertEqual(
         [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
     self.assertEqual([e[0] for e in expected_results]      ,  tokens["offset_mapping"]					)







  @slow
  def   snake_case		(   self					):







   """simple docstring"""
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
     lowerCamelCase_								=  self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,  **UpperCamelCase					)
     lowerCamelCase_								=  self.tokenizer_class.from_pretrained(UpperCamelCase      ,  **UpperCamelCase					)
     lowerCamelCase_								=  [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

     lowerCamelCase_								=  tokenizer_r.batch_encode_candidates(UpperCamelCase      ,  max_length=10      ,  return_tensors="np"					)
     lowerCamelCase_								=  tokenizer_p.batch_encode_candidates(UpperCamelCase      ,  max_length=10      ,  return_tensors="np"					)

     lowerCamelCase_								=  (2, 2, 10)

     self.assertEqual(encoded_sentence_r["input_ids"].shape      ,  UpperCamelCase					)
     self.assertEqual(encoded_sentence_r["attention_mask"].shape      ,  UpperCamelCase					)
     self.assertEqual(encoded_sentence_r["token_type_ids"].shape      ,  UpperCamelCase					)

     self.assertEqual(encoded_sentence_p["input_ids"].shape      ,  UpperCamelCase					)
     self.assertEqual(encoded_sentence_p["attention_mask"].shape      ,  UpperCamelCase					)
     self.assertEqual(encoded_sentence_p["token_type_ids"].shape      ,  UpperCamelCase					)







'''simple docstring'''

import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
     import tensorflow as tf

     from transformers.activations_tf import get_tf_activation





@require_tf
class 					snake_case   (   unittest.TestCase      ):





  """simple docstring"""




  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  tf.constant([-100, -1.0, -0.1, 0, 0.1, 1.0, 100.0]					)
   lowerCamelCase_								=  get_tf_activation("gelu"					)
   lowerCamelCase_								=  get_tf_activation("gelu_10"					)

   lowerCamelCase_								=  gelu(UpperCamelCase					)
   lowerCamelCase_								=  geluaa(UpperCamelCase					)

   lowerCamelCase_								=  tf.where(y_gelu_aa < 10.0      ,  1.0      ,  0.0					)

   self.assertEqual(tf.math.reduce_max(UpperCamelCase					).numpy().item()      ,  10.0					)
   self.assertTrue(np.allclose(y_gelu * clipped_mask      ,  y_gelu_aa * clipped_mask					)					)







  def   snake_case		(   self					):







   """simple docstring"""
   get_tf_activation("gelu"					)
   get_tf_activation("gelu_10"					)
   get_tf_activation("gelu_fast"					)
   get_tf_activation("gelu_new"					)
   get_tf_activation("glu"					)
   get_tf_activation("mish"					)
   get_tf_activation("quick_gelu"					)
   get_tf_activation("relu"					)
   get_tf_activation("sigmoid"					)
   get_tf_activation("silu"					)
   get_tf_activation("swish"					)
   get_tf_activation("tanh"					)
   with self.assertRaises(UpperCamelCase					):
    get_tf_activation("bogus"					)
   with self.assertRaises(UpperCamelCase					):
    get_tf_activation(UpperCamelCase					)



