



'''simple docstring'''





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class __A      (  A       ,       unittest.TestCase						):



 '''simple docstring'''







 __lowerCamelCase   :	int										=	ElectraTokenizer
 __lowerCamelCase   :	Dict										=	ElectraTokenizerFast
 __lowerCamelCase   :	str										=	True
 __lowerCamelCase   :	Union[str, Any]										=	True
 __lowerCamelCase   :	int										=	filter_non_english



 def  a__  (self	)   ->					List[Any]:


      """simple docstring"""



      super().setUp()

      _a     =  [
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      _a     =  os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['''vocab_file''']	)
      with open(self.vocab_file			,       '''w'''			,       encoding='''utf-8'''	) as vocab_writer:
           vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)



 def  a__  (self			,       A	)   ->					Tuple:


      """simple docstring"""



      _a     =  '''UNwant\u00E9d,running'''
      _a     =  '''unwanted, running'''
      return input_text, output_text



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  self.tokenizer_class(self.vocab_file	)

      _a     =  tokenizer.tokenize('''UNwant\u00E9d,running'''	)
      self.assertListEqual(A			,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A	)			,       [9, 6, 7, 12, 10, 11]	)



 def  a__  (self	)   ->					List[str]:


      """simple docstring"""



      if not self.test_rust_tokenizer:
           return

      _a     =  self.get_tokenizer()
      _a     =  self.get_rust_tokenizer()

      _a     =  '''UNwant\u00E9d,running'''

      _a     =  tokenizer.tokenize(A	)
      _a     =  rust_tokenizer.tokenize(A	)
      self.assertListEqual(A			,       A	)

      _a     =  tokenizer.encode(A			,       add_special_tokens=A	)
      _a     =  rust_tokenizer.encode(A			,       add_special_tokens=A	)
      self.assertListEqual(A			,       A	)

      _a     =  self.get_rust_tokenizer()
      _a     =  tokenizer.encode(A	)
      _a     =  rust_tokenizer.encode(A	)
      self.assertListEqual(A			,       A	)

      # With lower casing
      _a     =  self.get_tokenizer(do_lower_case=A	)
      _a     =  self.get_rust_tokenizer(do_lower_case=A	)

      _a     =  '''UNwant\u00E9d,running'''

      _a     =  tokenizer.tokenize(A	)
      _a     =  rust_tokenizer.tokenize(A	)
      self.assertListEqual(A			,       A	)

      _a     =  tokenizer.encode(A			,       add_special_tokens=A	)
      _a     =  rust_tokenizer.encode(A			,       add_special_tokens=A	)
      self.assertListEqual(A			,       A	)

      _a     =  self.get_rust_tokenizer()
      _a     =  tokenizer.encode(A	)
      _a     =  rust_tokenizer.encode(A	)
      self.assertListEqual(A			,       A	)



 def  a__  (self	)   ->					Optional[int]:


      """simple docstring"""



      _a     =  BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)			,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)



 def  a__  (self	)   ->					int:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)			,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)			,       ['''hello''']	)



 def  a__  (self	)   ->					str:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A			,       strip_accents=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)			,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)			,       ['''h\u00E9llo''']	)



 def  a__  (self	)   ->					Optional[Any]:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A			,       strip_accents=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)			,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)			,       ['''hello''']	)



 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)			,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)			,       ['''hello''']	)



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)			,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)



 def  a__  (self	)   ->					List[Any]:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A			,       strip_accents=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)			,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)



 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A			,       strip_accents=A	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)			,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)



 def  a__  (self	)   ->					Tuple:


      """simple docstring"""



      _a     =  BasicTokenizer(do_lower_case=A			,       never_split=['''[UNK]''']	)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)			,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      _a     =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      _a     =  {}
      for i, token in enumerate(A	):
           _a     =  i
      _a     =  WordpieceTokenizer(vocab=A			,       unk_token='''[UNK]'''	)

      self.assertListEqual(tokenizer.tokenize(''''''	)			,       []	)

      self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)			,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

      self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)			,       ['''[UNK]''', '''runn''', '''##ing''']	)



 def  a__  (self	)   ->					Any:


      """simple docstring"""



      self.assertTrue(_is_whitespace(''' '''	)	)
      self.assertTrue(_is_whitespace('''\t'''	)	)
      self.assertTrue(_is_whitespace('''\r'''	)	)
      self.assertTrue(_is_whitespace('''\n'''	)	)
      self.assertTrue(_is_whitespace('''\u00A0'''	)	)

      self.assertFalse(_is_whitespace('''A'''	)	)
      self.assertFalse(_is_whitespace('''-'''	)	)



 def  a__  (self	)   ->					int:


      """simple docstring"""



      self.assertTrue(_is_control('''\u0005'''	)	)

      self.assertFalse(_is_control('''A'''	)	)
      self.assertFalse(_is_control(''' '''	)	)
      self.assertFalse(_is_control('''\t'''	)	)
      self.assertFalse(_is_control('''\r'''	)	)



 def  a__  (self	)   ->					List[Any]:


      """simple docstring"""



      self.assertTrue(_is_punctuation('''-'''	)	)
      self.assertTrue(_is_punctuation('''$'''	)	)
      self.assertTrue(_is_punctuation('''`'''	)	)
      self.assertTrue(_is_punctuation('''.'''	)	)

      self.assertFalse(_is_punctuation('''A'''	)	)
      self.assertFalse(_is_punctuation(''' '''	)	)



 def  a__  (self	)   ->					Optional[Any]:


      """simple docstring"""



      _a     =  self.get_tokenizer()
      _a     =  self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(A	) for t in ['''Test''', '''\xad''', '''test''']]			,       [['''[UNK]'''], [], ['''[UNK]''']]	)

      self.assertListEqual(
          [rust_tokenizer.tokenize(A	) for t in ['''Test''', '''\xad''', '''test''']]			,       [['''[UNK]'''], [], ['''[UNK]''']]	)



 @slow
 def  a__  (self	)   ->					Any:


      """simple docstring"""



      _a     =  self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''	)

      _a     =  tokenizer.encode('''sequence builders'''			,       add_special_tokens=A	)
      _a     =  tokenizer.encode('''multi-sequence build'''			,       add_special_tokens=A	)

      _a     =  tokenizer.build_inputs_with_special_tokens(A	)
      _a     =  tokenizer.build_inputs_with_special_tokens(A			,       A	)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]



 def  a__  (self	)   ->					Dict:


      """simple docstring"""



      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                _a     =  self.rust_tokenizer_class.from_pretrained(A			,       **A	)

                _a     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                _a     =  tokenizer_r.encode_plus(
                    A			,       return_attention_mask=A			,       return_token_type_ids=A			,       return_offsets_mapping=A			,       add_special_tokens=A			,       )

                _a     =  tokenizer_r.do_lower_case if hasattr(A			,       '''do_lower_case'''	) else False
                _a     =  (
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''A'''),
                        ((1, 2), ''','''),
                        ((3, 5), '''na'''),
                        ((5, 6), '''##ï'''),
                        ((6, 8), '''##ve'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''Allen'''),
                        ((21, 23), '''##NL'''),
                        ((23, 24), '''##P'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''a'''),
                        ((1, 2), ''','''),
                        ((3, 8), '''naive'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''allen'''),
                        ((21, 23), '''##nl'''),
                        ((23, 24), '''##p'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]			,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
                self.assertEqual([e[0] for e in expected_results]			,       tokens['''offset_mapping''']	)



 def  a__  (self	)   ->					Optional[Any]:


      """simple docstring"""



      _a     =  ['''的''', '''人''', '''有''']
      _a     =  ''''''.join(A	)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                _a     =  True
                _a     =  self.tokenizer_class.from_pretrained(A			,       **A	)
                _a     =  self.rust_tokenizer_class.from_pretrained(A			,       **A	)

                _a     =  tokenizer_p.encode(A			,       add_special_tokens=A	)
                _a     =  tokenizer_r.encode(A			,       add_special_tokens=A	)

                _a     =  tokenizer_r.convert_ids_to_tokens(A	)
                _a     =  tokenizer_p.convert_ids_to_tokens(A	)

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(A			,       A	)
                self.assertListEqual(A			,       A	)

                _a     =  False
                _a     =  self.rust_tokenizer_class.from_pretrained(A			,       **A	)
                _a     =  self.tokenizer_class.from_pretrained(A			,       **A	)

                _a     =  tokenizer_r.encode(A			,       add_special_tokens=A	)
                _a     =  tokenizer_p.encode(A			,       add_special_tokens=A	)

                _a     =  tokenizer_r.convert_ids_to_tokens(A	)
                _a     =  tokenizer_p.convert_ids_to_tokens(A	)

                # it is expected that only the first Chinese character is not preceded by "##".
                _a     =  [
                    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(A	)
                ]
                self.assertListEqual(A			,       A	)
                self.assertListEqual(A			,       A	)








'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  lowercase_			=  None


lowercase_			=  logging.get_logger(__name__)

lowercase_			=  {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowercase_			=  {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

lowercase_			=  {
    "facebook/mbart-large-50-one-to-many-mmt": 1_024,
}

# fmt: off
lowercase_			=  ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]






class __A      (  A						):



 '''simple docstring'''







 __lowerCamelCase   :	List[str]										=	VOCAB_FILES_NAMES
 __lowerCamelCase   :	int										=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCamelCase   :	Optional[int]										=	PRETRAINED_VOCAB_FILES_MAP
 __lowerCamelCase   :	Any										=	['input_ids', 'attention_mask']
 __lowerCamelCase   :	Optional[Any]										=	MBartaaTokenizer

 __lowerCamelCase   :	List[int]										=	[]
 __lowerCamelCase   :	List[int]										=	[]



 def __init__(self			,       A=None			,       A=None			,       A=None			,       A=None			,       A="</s>"			,       A="</s>"			,       A="<s>"			,       A="<unk>"			,       A="<pad>"			,       A="<mask>"			,       **A			,       )   ->					List[str]:


      """simple docstring"""



      _a     =  AddedToken(A			,       lstrip=A			,       rstrip=A	) if isinstance(A			,       A	) else mask_token

      _a     =  kwargs.get('''additional_special_tokens'''			,       []	)
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          A			,       src_lang=A			,       tgt_lang=A			,       tokenizer_file=A			,       eos_token=A			,       sep_token=A			,       cls_token=A			,       unk_token=A			,       pad_token=A			,       mask_token=A			,       **A			,       )

      _a     =  vocab_file
      _a     =  False if not self.vocab_file else True

      _a     =  {
          lang_code: self.convert_tokens_to_ids(A	) for lang_code in FAIRSEQ_LANGUAGE_CODES
      }

      _a     =  src_lang if src_lang is not None else '''en_XX'''
      _a     =  tgt_lang
      _a     =  self.lang_code_to_id[self._src_lang]
      self.set_src_lang_special_tokens(self._src_lang	)



 @property
 def  a__  (self	)   ->					str:


      """simple docstring"""



      return self._src_lang



 @src_lang.setter
 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  new_src_lang
      self.set_src_lang_special_tokens(self._src_lang	)



 def  a__  (self			,       A			,       A = None	)   ->					List[int]:


      """simple docstring"""



      if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



 def  a__  (self			,       A			,       A = "en_XX"			,       A = None			,       A = "ro_RO"			,       **A			,       )   ->					BatchEncoding:


      """simple docstring"""



      _a     =  src_lang
      _a     =  tgt_lang
      return super().prepare_seqaseq_batch(A			,       A			,       **A	)



 def  a__  (self	)   ->					Union[str, Any]:


      """simple docstring"""



      return self.set_src_lang_special_tokens(self.src_lang	)



 def  a__  (self	)   ->					Union[str, Any]:


      """simple docstring"""



      return self.set_tgt_lang_special_tokens(self.tgt_lang	)



 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  [self.cur_lang_code_id]
      _a     =  [self.eos_token_id]

      _a     =  self.convert_ids_to_tokens(self.prefix_tokens	)
      _a     =  self.convert_ids_to_tokens(self.suffix_tokens	)

      _a     =  processors.TemplateProcessing(
          single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str			,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str			,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,       self.prefix_tokens + self.suffix_tokens	)	)			,       )



 def  a__  (self			,       A	)   ->					None:


      """simple docstring"""



      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  [self.cur_lang_code_id]
      _a     =  [self.eos_token_id]

      _a     =  self.convert_ids_to_tokens(self.prefix_tokens	)
      _a     =  self.convert_ids_to_tokens(self.suffix_tokens	)

      _a     =  processors.TemplateProcessing(
          single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str			,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str			,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,       self.prefix_tokens + self.suffix_tokens	)	)			,       )



 def  a__  (self			,       A			,       A			,       A			,       A			,       **A	)   ->					Optional[int]:


      """simple docstring"""



      if src_lang is None or tgt_lang is None:
           raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''	)
      _a     =  src_lang
      _a     =  self(A			,       add_special_tokens=A			,       return_tensors=A			,       **A	)
      _a     =  self.convert_tokens_to_ids(A	)
      _a     =  tgt_lang_id
      return inputs



 def  a__  (self			,       A			,       A = None	)   ->					Tuple[str]:


      """simple docstring"""



      if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.'''	)

      if not os.path.isdir(A	):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
           return
      _a     =  os.path.join(
          A			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(A	):
           copyfile(self.vocab_file			,       A	)

      return (out_vocab_file,)



