





"""simple docstring"""





import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer





class       snake_case		(       tf.keras.layers.Layer       ):

      def __init__(      self       :Optional[int]						,							_lowerCamelCase       :List						,							_lowerCamelCase       :bool						,							_lowerCamelCase       :int = None						,							_lowerCamelCase       :int = None						,							_lowerCamelCase       :int = None						,							_lowerCamelCase       :str = "longest"						,							_lowerCamelCase       :bool = True						,							_lowerCamelCase       :int = 5_1_2						,							_lowerCamelCase       :int = None						,							_lowerCamelCase       :bool = True						,							_lowerCamelCase       :bool = True						,							_lowerCamelCase       :bool = True						,							**_lowerCamelCase       :Tuple						,							):
       super().__init__()
       if use_fast_bert_tokenizer:
        __SCREAMING_SNAKE_CASE			:      int 							=	FastBertTokenizer(
            _lowerCamelCase						,							token_out_type=tf.intaa						,							lower_case_nfd_strip_accents=_lowerCamelCase						,							**_lowerCamelCase						)
       else:
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	tf.lookup.StaticVocabularyTable(
            tf.lookup.KeyValueTensorInitializer(
                keys=_lowerCamelCase						,							key_dtype=tf.string						,							values=tf.range(tf.size(_lowerCamelCase						,							out_type=tf.intaa						)						,							dtype=tf.intaa						)						,							value_dtype=tf.intaa						,							)						,							num_oov_buckets=1						,							)
        __SCREAMING_SNAKE_CASE			:      Dict 							=	BertTokenizerLayer(
            _lowerCamelCase						,							token_out_type=tf.intaa						,							lower_case=_lowerCamelCase						,							**_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	vocab_list
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	do_lower_case
       __SCREAMING_SNAKE_CASE			:      Dict 							=	cls_token_id or vocab_list.index('''[CLS]'''						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	sep_token_id or vocab_list.index('''[SEP]'''						)
       __SCREAMING_SNAKE_CASE			:      Dict 							=	pad_token_id or vocab_list.index('''[PAD]'''						)
       __SCREAMING_SNAKE_CASE			:      Dict 							=	ShrinkLongestTrimmer(max_length - 3						,							axis=1						)  # Allow room for special tokens
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	max_length
       __SCREAMING_SNAKE_CASE			:      int 							=	padding
       __SCREAMING_SNAKE_CASE			:      int 							=	truncation
       __SCREAMING_SNAKE_CASE			:      str 							=	pad_to_multiple_of
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	return_token_type_ids
       __SCREAMING_SNAKE_CASE			:      str 							=	return_attention_mask

      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :Any						,							_lowerCamelCase       :"PreTrainedTokenizerBase"						,							**_lowerCamelCase       :Any						):  # noqa: F821
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	kwargs.pop('''do_lower_case'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      str 							=	tokenizer.do_lower_case if do_lower_case is None else do_lower_case
       __SCREAMING_SNAKE_CASE			:      Any 							=	kwargs.pop('''cls_token_id'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      int 							=	tokenizer.cls_token_id if cls_token_id is None else cls_token_id
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	kwargs.pop('''sep_token_id'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	tokenizer.sep_token_id if sep_token_id is None else sep_token_id
       __SCREAMING_SNAKE_CASE			:      Any 							=	kwargs.pop('''pad_token_id'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	tokenizer.pad_token_id if pad_token_id is None else pad_token_id

       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	tokenizer.get_vocab()
       __SCREAMING_SNAKE_CASE			:      Any 							=	sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]						,							key=lambda _lowerCamelCase						: x[1]						)
       __SCREAMING_SNAKE_CASE			:      int 							=	[entry[0] for entry in vocab]
       return cls(
           vocab_list=_lowerCamelCase						,							do_lower_case=_lowerCamelCase						,							cls_token_id=_lowerCamelCase						,							sep_token_id=_lowerCamelCase						,							pad_token_id=_lowerCamelCase						,							**_lowerCamelCase						,							)

      @classmethod
      def 						SCREAMING_SNAKE_CASE_      (      cls       :Tuple						,							_lowerCamelCase       :Union[str, os.PathLike]						,							*_lowerCamelCase       :List[str]						,							**_lowerCamelCase       :Optional[int]						):
       try:
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	BertTokenizer.from_pretrained(_lowerCamelCase						,							*_lowerCamelCase						,							**_lowerCamelCase						)
       except:  # noqa: E722
        from .tokenization_bert_fast import BertTokenizerFast

        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	BertTokenizerFast.from_pretrained(_lowerCamelCase						,							*_lowerCamelCase						,							**_lowerCamelCase						)
       return cls.from_tokenizer(_lowerCamelCase						,							**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						,							_lowerCamelCase       :int						):
       if self.do_lower_case:
        __SCREAMING_SNAKE_CASE			:      Any 							=	case_fold_utfa(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.tf_tokenizer.tokenize(_lowerCamelCase						)
       return tokens.merge_dims(1						,							-1						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :Optional[Any]						,							_lowerCamelCase       :List[Any]=None						,							_lowerCamelCase       :List[str]=None						,							_lowerCamelCase       :str=None						,							_lowerCamelCase       :List[Any]=None						,							_lowerCamelCase       :Tuple=None						,							_lowerCamelCase       :int=None						,							_lowerCamelCase       :str=None						,							):
       if padding is None:
        __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.padding
       if padding not in ("longest", "max_length"):
        raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''						)
       if max_length is not None and text_pair is not None:
        # Because we have to instantiate a Trimmer to do it properly
        raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''						)
       if max_length is None:
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.max_length
       if truncation is None:
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self.truncation
       if pad_to_multiple_of is None:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	self.pad_to_multiple_of
       if return_token_type_ids is None:
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.return_token_type_ids
       if return_attention_mask is None:
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	self.return_attention_mask
       if not isinstance(_lowerCamelCase						,							tf.Tensor						):
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tf.convert_to_tensor(_lowerCamelCase						)
       if text_pair is not None and not isinstance(_lowerCamelCase						,							tf.Tensor						):
        __SCREAMING_SNAKE_CASE			:      Any 							=	tf.convert_to_tensor(_lowerCamelCase						)
       if text_pair is not None:
        if text.shape.rank > 1:
         raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''						)
        if text_pair.shape.rank > 1:
         raise ValueError('''text_pair should not be multidimensional!'''						)
       if text.shape.rank == 2:
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	text[:, 0], text[:, 1]
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.unpaired_tokenize(_lowerCamelCase						)
       if text_pair is None:  # Unpaired text
        if truncation:
         __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	text[:, : max_length - 2]  # Allow room for special tokens
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	combine_segments(
            (text,)						,							start_of_sequence_id=self.cls_token_id						,							end_of_segment_id=self.sep_token_id						)
       else:  # Paired text
        __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.unpaired_tokenize(_lowerCamelCase						)
        if truncation:
         __SCREAMING_SNAKE_CASE			:      Any 							=	self.paired_trimmer.trim([text, text_pair]						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	combine_segments(
            (text, text_pair)						,							start_of_sequence_id=self.cls_token_id						,							end_of_segment_id=self.sep_token_id						)
       if padding == "longest":
        __SCREAMING_SNAKE_CASE			:      str 							=	input_ids.bounding_shape(axis=1						)
        if pad_to_multiple_of is not None:
         # No ceiling division in tensorflow, so we negate floordiv instead
         __SCREAMING_SNAKE_CASE			:      Dict 							=	pad_to_multiple_of * (-tf.math.floordiv(-pad_length						,							_lowerCamelCase						))
       else:
        __SCREAMING_SNAKE_CASE			:      int 							=	max_length

       __SCREAMING_SNAKE_CASE			:      Any 							=	pad_model_inputs(_lowerCamelCase						,							max_seq_length=_lowerCamelCase						,							pad_value=self.pad_token_id						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	{'''input_ids''': input_ids}
       if return_attention_mask:
        __SCREAMING_SNAKE_CASE			:      str 							=	attention_mask
       if return_token_type_ids:
        __SCREAMING_SNAKE_CASE			:      Any 							=	pad_model_inputs(
            _lowerCamelCase						,							max_seq_length=_lowerCamelCase						,							pad_value=self.pad_token_id						)
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	token_type_ids
       return output



      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       return {
           "vocab_list": self.vocab_list,
           "do_lower_case": self.do_lower_case,
           "cls_token_id": self.cls_token_id,
           "sep_token_id": self.sep_token_id,
           "pad_token_id": self.pad_token_id,
       }








"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case		(       __UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			PreTrainedTokenizerFast
      lowerCamelCase__      =			False
      lowerCamelCase__      =			True
      lowerCamelCase__      =			'''tokenizer_file'''

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	True

       __SCREAMING_SNAKE_CASE			:      int 							=	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       __SCREAMING_SNAKE_CASE			:      str 							=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
       tokenizer.save_pretrained(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      int 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	tmpdirname_orig

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      Dict 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tmpdirname_orig



      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	('''a''' for _ in range(1_0_0_0						))
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase						,							length=1_0_0_0						,							vocab_size=5_0						)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	new_tokenizer.encode('''a🤗'''						)
       self.assertEqual(_lowerCamelCase						,							[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained('''bert-base-cased'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	json.loads(tokenizer._tokenizer.to_str()						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	len(_lowerCamelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	['''tokenizer.4.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        json.dump(_lowerCamelCase						,							open(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							'''w'''						)						)

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __SCREAMING_SNAKE_CASE			:      Dict 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						) + 1						)
        __SCREAMING_SNAKE_CASE			:      Dict 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							os.path.join(_lowerCamelCase						,							'''tokenizer.42.0.0.json'''						)						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	['''tokenizer.42.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						)						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	'''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       __SCREAMING_SNAKE_CASE			:      int 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_7						)
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	json.loads(tokenizer._tokenizer.to_str()						)
       self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''3.0.0'''
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_6						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	json.loads(old_tokenizer._tokenizer.to_str()						)
       self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	'''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[executor.submit(self.fetch						,							_lowerCamelCase						,							_lowerCamelCase						) for i in range(1_0						)]
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	[future.result() for future in futures]
        self.assertEqual(_lowerCamelCase						,							[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0						)]						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Optional[int]						):
       return tokenizer.encode(_lowerCamelCase						,							truncation='''longest_first'''						,							padding='''longest'''						)

