



import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 						_snake_case (     lowerCamelCase   ,unittest.TestCase			):





 """simple docstring"""



 lowerCamelCase_       =     GPTNeoXJapaneseTokenizer
 lowerCamelCase_       =     False
 lowerCamelCase_       =     {'''do_clean_text''': False, '''add_prefix_space''': False}
 def lowercase_						(  self   )		->							Dict:




   """simple docstring"""







   super().setUp()

   _A          =      [
       '''こん''',
       '''こんに''',
       '''にちは''',
       '''ばんは''',
       '''世界,㔺界''',
       '''、''',
       '''。''',
       '''<BR>''',
       '''<SP>''',
       '''<TAB>''',
       '''<URL>''',
       '''<EMAIL>''',
       '''<TEL>''',
       '''<DATE>''',
       '''<PRICE>''',
       '''<BLOCK>''',
       '''<KIGOU>''',
       '''<U2000U2BFF>''',
       '''<|emoji1|>''',
       '''<unk>''',
       '''<|startoftext|>''',
       '''<|endoftext|>''',
   ]
   _A          =      {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
   _A          =      {'''unk_token''': '''<unk>'''}

   _A          =      os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES['''vocab_file''']   )
   _A          =      os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES['''emoji_file''']   )
   with open(self.vocab_file		,      '''w'''		,      encoding='''utf-8'''   ) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
   with open(self.emoji_file		,      '''w'''   ) as emoji_writer:
     emoji_writer.write(json.dumps(a   )   )
 def lowercase_						(  self		,      **a   )		->							Any:




   """simple docstring"""







   kwargs.update(self.special_tokens_map   )
   return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname		,      **a   )
 def lowercase_						(  self		,      a   )		->							Any:




   """simple docstring"""







   _A          =      '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
   _A          =      '''こんにちは、世界。 \nこんばんは、世界。😀'''
   return input_text, output_text
 def lowercase_						(  self		,      a   )		->							int:




   """simple docstring"""







   _A      ,				_A          =      self.get_input_output_texts(a   )
   _A          =      tokenizer.encode(a		,      add_special_tokens=a   )
   _A          =      tokenizer.decode(a		,      clean_up_tokenization_spaces=a   )
   return text, ids
 def lowercase_						(  self   )		->							Tuple:




   """simple docstring"""







   pass  # TODO add if relevant
 def lowercase_						(  self   )		->							Dict:




   """simple docstring"""







   pass  # TODO add if relevant
 def lowercase_						(  self   )		->							List[str]:




   """simple docstring"""







   pass  # TODO add if relevant
 def lowercase_						(  self   )		->							Optional[int]:




   """simple docstring"""







   _A          =      self.get_tokenizer()

   # Testing tokenization
   _A          =      '''こんにちは、世界。　こんばんは、㔺界。'''
   _A          =      ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
   _A          =      tokenizer.tokenize(a   )
   self.assertListEqual(a		,      a   )

   # Testing conversion to ids without special tokens
   _A          =      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
   _A          =      tokenizer.convert_tokens_to_ids(a   )
   self.assertListEqual(a		,      a   )

   # Testing conversion to ids with special tokens
   _A          =      tokens + [tokenizer.unk_token]
   _A          =      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
   _A          =      tokenizer.convert_tokens_to_ids(a   )
   self.assertListEqual(a		,      a   )
 @slow
 def lowercase_						(  self   )		->							Any:




   """simple docstring"""







   _A          =      self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b'''   )

   _A          =      tokenizer.encode('''ありがとう。'''		,      add_special_tokens=a   )
   _A          =      tokenizer.encode('''どういたしまして。'''		,      add_special_tokens=a   )

   _A          =      tokenizer.build_inputs_with_special_tokens(a   )
   _A          =      tokenizer.build_inputs_with_special_tokens(a		,      a   )

   assert encoded_sentence == ids_a
   assert encoded_pair == ids_a + ids_a
 def lowercase_						(  self   )		->							str:




   """simple docstring"""







   pass



 def lowercase_						(  self   )		->							Any:




   """simple docstring"""







   pass




import importlib.metadata
import subprocess
import sys




def       UpperCAmelCase__	(							__snake_case = False							)				->	List[Any]:
  _A          =      False
  if "IPython" in sys.modules:
    _A          =      '''google.colab''' in str(sys.modules['''IPython'''].get_ipython()							)

  if in_colab:
    if upgrade:
      _A          =      ['''pip''', '''install''', '''-U''', '''torch''']
      subprocess.run(__snake_case  ,			check=__snake_case							)
    # get the current version of torch
    _A          =      importlib.metadata.version('''torch'''							)
    _A          =      torch_version[: torch_version.rindex('''.'''							)]
    _A          =      F'''https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl'''
    _A          =      ['''pip''', '''install''', xla_wheel]
    subprocess.run(__snake_case  ,			check=__snake_case							)
  else:
    raise RuntimeError('''`install_xla` utility works only on google colab.'''							)