'''simple docstring'''




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      lowerCamelCase_     (snake_case__							,			unittest.TestCase   ):
      '''simple docstring'''

      __UpperCamelCase:  Optional[int]      =						ElectraTokenizer
      __UpperCamelCase:  Optional[Any]      =						ElectraTokenizerFast
      __UpperCamelCase:  List[Any]      =						True
      __UpperCamelCase:  int      =						True
      __UpperCamelCase:  str      =						filter_non_english




      def 					_A     (	self						:    Any	):
             super().setUp()

             _UpperCAmelCase					:  str											=   [
                 "[UNK]",
                 "[CLS]",
                 "[SEP]",
                 "[PAD]",
                 "[MASK]",
                 "want",
                 "##want",
                 "##ed",
                 "wa",
                 "un",
                 "runn",
                 "##ing",
                 ",",
                 "low",
                 "lowest",
             ]
             _UpperCAmelCase					:  Optional[Any]											=   os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["vocab_file"]	)
             with open(self.vocab_file						, "w"						, encoding="utf-8"	) as vocab_writer:
                    vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)




      def 					_A     (	self						:    Optional[int]						, A						:    List[Any]	):
             _UpperCAmelCase					:  int											=   "UNwant\u00E9d,running"
             _UpperCAmelCase					:  str											=   "unwanted, running"
             return input_text, output_text




      def 					_A     (	self						:    Tuple	):
             _UpperCAmelCase					:  List[str]											=   self.tokenizer_class(self.vocab_file	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.tokenize("UNwant\u00E9d,running"	)
             self.assertListEqual(A						, ["un", "##want", "##ed", ",", "runn", "##ing"]	)
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A	)						, [9, 6, 7, 12, 10, 11]	)




      def 					_A     (	self						:    Any	):
             if not self.test_rust_tokenizer:
                    return

             _UpperCAmelCase					:  Union[str, Any]											=   self.get_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   self.get_rust_tokenizer()

             _UpperCAmelCase					:  Any											=   "UNwant\u00E9d,running"

             _UpperCAmelCase					:  Optional[int]											=   tokenizer.tokenize(A	)
             _UpperCAmelCase					:  List[Any]											=   rust_tokenizer.tokenize(A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[int]											=   rust_tokenizer.encode(A						, add_special_tokens=A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  int											=   self.get_rust_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A	)
             _UpperCAmelCase					:  List[str]											=   rust_tokenizer.encode(A	)
             self.assertListEqual(A						, A	)

             # With lower casing
             _UpperCAmelCase					:  Dict											=   self.get_tokenizer(do_lower_case=A	)
             _UpperCAmelCase					:  Dict											=   self.get_rust_tokenizer(do_lower_case=A	)

             _UpperCAmelCase					:  List[Any]											=   "UNwant\u00E9d,running"

             _UpperCAmelCase					:  List[Any]											=   tokenizer.tokenize(A	)
             _UpperCAmelCase					:  Dict											=   rust_tokenizer.tokenize(A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.encode(A						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[Any]											=   rust_tokenizer.encode(A						, add_special_tokens=A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  Dict											=   self.get_rust_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A	)
             _UpperCAmelCase					:  Any											=   rust_tokenizer.encode(A	)
             self.assertListEqual(A						, A	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  Tuple											=   BasicTokenizer()

             self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)						, ["ah", "\u535A", "\u63A8", "zz"]	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  List[Any]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						, ["hello", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  str											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hällo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["h\u00E9llo"]	)




      def 					_A     (	self						:    List[str]	):
             _UpperCAmelCase					:  Any											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hallo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    int	):
             _UpperCAmelCase					:  Optional[int]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hallo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    int	):
             _UpperCAmelCase					:  Union[str, Any]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						, ["HeLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Any	):
             _UpperCAmelCase					:  int											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["HäLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Optional[int]	):
             _UpperCAmelCase					:  List[str]											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["HaLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  Any											=   BasicTokenizer(do_lower_case=A						, never_split=["[UNK]"]	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)						, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)




      def 					_A     (	self						:    Optional[Any]	):
             _UpperCAmelCase					:  List[str]											=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

             _UpperCAmelCase					:  Union[str, Any]											=   {}
             for i, token in enumerate(A	):
                    _UpperCAmelCase					:  str											=   i
             _UpperCAmelCase					:  Tuple											=   WordpieceTokenizer(vocab=A						, unk_token="[UNK]"	)

             self.assertListEqual(tokenizer.tokenize(""	)						, []	)

             self.assertListEqual(tokenizer.tokenize("unwanted running"	)						, ["un", "##want", "##ed", "runn", "##ing"]	)

             self.assertListEqual(tokenizer.tokenize("unwantedX running"	)						, ["[UNK]", "runn", "##ing"]	)




      def 					_A     (	self						:    int	):
             self.assertTrue(_is_whitespace(" "	)	)
             self.assertTrue(_is_whitespace("\t"	)	)
             self.assertTrue(_is_whitespace("\r"	)	)
             self.assertTrue(_is_whitespace("\n"	)	)
             self.assertTrue(_is_whitespace("\u00A0"	)	)

             self.assertFalse(_is_whitespace("A"	)	)
             self.assertFalse(_is_whitespace("-"	)	)




      def 					_A     (	self						:    List[Any]	):
             self.assertTrue(_is_control("\u0005"	)	)

             self.assertFalse(_is_control("A"	)	)
             self.assertFalse(_is_control(" "	)	)
             self.assertFalse(_is_control("\t"	)	)
             self.assertFalse(_is_control("\r"	)	)




      def 					_A     (	self						:    Optional[Any]	):
             self.assertTrue(_is_punctuation("-"	)	)
             self.assertTrue(_is_punctuation("$"	)	)
             self.assertTrue(_is_punctuation("`"	)	)
             self.assertTrue(_is_punctuation("."	)	)

             self.assertFalse(_is_punctuation("A"	)	)
             self.assertFalse(_is_punctuation(" "	)	)




      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  Any											=   self.get_tokenizer()
             _UpperCAmelCase					:  str											=   self.get_rust_tokenizer()

             # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
             self.assertListEqual([tokenizer.tokenize(A	) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]]	)

             self.assertListEqual(
                 [rust_tokenizer.tokenize(A	) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]]	)




      @slow
      def 					_A     (	self						:    Tuple	):
             _UpperCAmelCase					:  List[str]											=   self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode("sequence builders"						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[Any]											=   tokenizer.encode("multi-sequence build"						, add_special_tokens=A	)

             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.build_inputs_with_special_tokens(A	)
             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.build_inputs_with_special_tokens(A						, A	)

             assert encoded_sentence == [101] + text + [102]
             assert encoded_pair == [101] + text + [102] + text_a + [102]




      def 					_A     (	self						:    Optional[Any]	):
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                           _UpperCAmelCase					:  Any											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[int]											=   F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                           _UpperCAmelCase					:  str											=   tokenizer_r.encode_plus(
                               A						, return_attention_mask=A						, return_token_type_ids=A						, return_offsets_mapping=A						, add_special_tokens=A						, )

                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.do_lower_case if hasattr(A						, "do_lower_case"	) else False
                           _UpperCAmelCase					:  str											=   (
                               [
                                   ((0, 0), tokenizer_r.cls_token),
                                   ((0, 1), "A"),
                                   ((1, 2), ","),
                                   ((3, 5), "na"),
                                   ((5, 6), "##ï"),
                                   ((6, 8), "##ve"),
                                   ((9, 15), tokenizer_r.mask_token),
                                   ((16, 21), "Allen"),
                                   ((21, 23), "##NL"),
                                   ((23, 24), "##P"),
                                   ((25, 33), "sentence"),
                                   ((33, 34), "."),
                                   ((0, 0), tokenizer_r.sep_token),
                               ]
                               if not do_lower_case
                               else [
                                   ((0, 0), tokenizer_r.cls_token),
                                   ((0, 1), "a"),
                                   ((1, 2), ","),
                                   ((3, 8), "naive"),
                                   ((9, 15), tokenizer_r.mask_token),
                                   ((16, 21), "allen"),
                                   ((21, 23), "##nl"),
                                   ((23, 24), "##p"),
                                   ((25, 33), "sentence"),
                                   ((33, 34), "."),
                                   ((0, 0), tokenizer_r.sep_token),
                               ]
                           )

                           self.assertEqual(
                               [e[1] for e in expected_results]						, tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
                           self.assertEqual([e[0] for e in expected_results]						, tokens["offset_mapping"]	)





      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  int											=   ["的", "人", "有"]
             _UpperCAmelCase					:  Dict											=   "".join(A	)
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                           _UpperCAmelCase					:  Optional[int]											=   True
                           _UpperCAmelCase					:  Tuple											=   self.tokenizer_class.from_pretrained(A						, **A	)
                           _UpperCAmelCase					:  Any											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[Any]											=   tokenizer_p.encode(A						, add_special_tokens=A	)
                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.encode(A						, add_special_tokens=A	)

                           _UpperCAmelCase					:  str											=   tokenizer_r.convert_ids_to_tokens(A	)
                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_p.convert_ids_to_tokens(A	)

                           # it is expected that each Chinese character is not preceded by "##"
                           self.assertListEqual(A						, A	)
                           self.assertListEqual(A						, A	)

                           _UpperCAmelCase					:  Tuple											=   False
                           _UpperCAmelCase					:  List[str]											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)
                           _UpperCAmelCase					:  Union[str, Any]											=   self.tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.encode(A						, add_special_tokens=A	)
                           _UpperCAmelCase					:  Union[str, Any]											=   tokenizer_p.encode(A						, add_special_tokens=A	)

                           _UpperCAmelCase					:  Tuple											=   tokenizer_r.convert_ids_to_tokens(A	)
                           _UpperCAmelCase					:  Optional[Any]											=   tokenizer_p.convert_ids_to_tokens(A	)

                           # it is expected that only the first Chinese character is not preceded by "##".
                           _UpperCAmelCase					:  Any											=   [
                               F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A	)
                           ]
                           self.assertListEqual(A						, A	)
                           self.assertListEqual(A						, A	)


'''simple docstring'''




import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter

class      lowerCamelCase_     (TensorFormatter[Mapping, np.ndarray, Mapping]   ):
      '''simple docstring'''

      def __init__(	self						:    str						, A						:    List[str]=None						, **A						:    Optional[Any]	):
             super().__init__(features=A	)
             _UpperCAmelCase					:  Tuple											=   np_array_kwargs




      def 					_A     (	self						:    str						, A						:    Tuple	):
             if isinstance(A						, A	):
                    if column and all(
                        isinstance(A						, np.ndarray	) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column	):
                           return np.stack(A	)
                    else:
                           # don't use np.array(column, dtype=object)
                           # since it fails in certain cases
                           # see https://stackoverflow.com/q/51005699
                           _UpperCAmelCase					:  List[str]											=   np.empty(len(A	)						, dtype=A	)
                           _UpperCAmelCase					:  int											=   column
                           return out
             return column




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             if isinstance(A						, (str, bytes, type(A	))	):
                    return value
             elif isinstance(A						, (np.character, np.ndarray)	) and np.issubdtype(value.dtype						, np.character	):
                    return value
             elif isinstance(A						, np.number	):
                    return value

             _UpperCAmelCase					:  List[str]											=   {}

             if isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.integer	):
                    _UpperCAmelCase					:  str											=   {"dtype": np.intaa}
             elif isinstance(A						, np.ndarray	) and np.issubdtype(value.dtype						, np.floating	):
                    _UpperCAmelCase					:  int											=   {"dtype": np.floataa}
             elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                    import PIL.Image

                    if isinstance(A						, PIL.Image.Image	):
                           return np.asarray(A						, **self.np_array_kwargs	)

             return np.asarray(A						, **{**default_dtype, **self.np_array_kwargs}	)




      def 					_A     (	self						:    Any						, A						:    Optional[Any]	):
             # support for torch, tf, jax etc.
             if config.TORCH_AVAILABLE and "torch" in sys.modules:
                    import torch

                    if isinstance(A						, torch.Tensor	):
                           return self._tensorize(data_struct.detach().cpu().numpy()[()]	)
             if hasattr(A						, "__array__"	) and not isinstance(A						, (np.ndarray, np.character, np.number)	):
                    _UpperCAmelCase					:  Dict											=   data_struct.__array__()
             # support for nested types like struct of list of struct
             if isinstance(A						, np.ndarray	):
                    if data_struct.dtype == object:
                           return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             if isinstance(A						, (list, tuple)	):
                    return self._consolidate([self.recursive_tensorize(A	) for substruct in data_struct]	)
             return self._tensorize(A	)




      def 					_A     (	self						:    List[Any]						, A						:    dict	):
             return map_nested(self._recursive_tensorize						, A						, map_list=A	)




      def 					_A     (	self						:    List[str]						, A						:    pa.Table	):
             _UpperCAmelCase					:  str											=   self.numpy_arrow_extractor().extract_row(A	)
             _UpperCAmelCase					:  Dict											=   self.python_features_decoder.decode_row(A	)
             return self.recursive_tensorize(A	)




      def 					_A     (	self						:    str						, A						:    pa.Table	):
             _UpperCAmelCase					:  List[str]											=   self.numpy_arrow_extractor().extract_column(A	)
             _UpperCAmelCase					:  Optional[Any]											=   self.python_features_decoder.decode_column(A						, pa_table.column_names[0]	)
             _UpperCAmelCase					:  Tuple											=   self.recursive_tensorize(A	)
             _UpperCAmelCase					:  Tuple											=   self._consolidate(A	)
             return column





      def 					_A     (	self						:    Union[str, Any]						, A						:    pa.Table	):
             _UpperCAmelCase					:  Any											=   self.numpy_arrow_extractor().extract_batch(A	)
             _UpperCAmelCase					:  Any											=   self.python_features_decoder.decode_batch(A	)
             _UpperCAmelCase					:  List[str]											=   self.recursive_tensorize(A	)
             for column_name in batch:
                    _UpperCAmelCase					:  Optional[Any]											=   self._consolidate(batch[column_name]	)
             return batch

