

import unittest

from cer import CER


__UpperCAmelCase	=			CER()







class        __lowercase     (      unittest.TestCase   ):

  def 				__lowercase    (					self						:     Union[str, Any]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[int]   				=   ["""White House"""]
   UpperCAmelCase__   :					Union[str, Any]   				=   ["""white house"""]
   # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
   UpperCAmelCase__   :					str   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.1_8_1_8_1_8_1_8_1_8      ) < 1e-6      )

  def 				__lowercase    (					self						:     Union[str, Any]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Union[str, Any]   				=   ["""were wolf"""]
   UpperCAmelCase__   :					Optional[int]   				=   ["""werewolf"""]
   # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
   UpperCAmelCase__   :					Union[str, Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.1_1_1_1_1_1_1      ) < 1e-6      )

   UpperCAmelCase__   :					Optional[int]   				=   ["""werewolf"""]
   UpperCAmelCase__   :					Tuple   				=   ["""weae     wolf"""]
   # S = 1, D = 1, I = 0, N = 8, CER = 0.25
   UpperCAmelCase__   :					Tuple   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.2_5      ) < 1e-6      )

   # consecutive whitespaces case 1
   UpperCAmelCase__   :					List[Any]   				=   ["""were wolf"""]
   UpperCAmelCase__   :					Union[str, Any]   				=   ["""were               wolf"""]
   # S = 0, D = 0, I = 0, N = 9, CER = 0
   UpperCAmelCase__   :					List[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.0      ) < 1e-6      )

   # consecutive whitespaces case 2
   UpperCAmelCase__   :					Optional[Any]   				=   ["""were   wolf"""]
   UpperCAmelCase__   :					Union[str, Any]   				=   ["""were               wolf"""]
   # S = 0, D = 0, I = 0, N = 9, CER = 0
   UpperCAmelCase__   :					Optional[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.0      ) < 1e-6      )

  def 				__lowercase    (					self						:     Tuple      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Tuple   				=   ["""werewolf"""]
   UpperCAmelCase__   :					Union[str, Any]   				=   ["""weaewolf"""]
   # S = 1, D = 0, I = 0, N = 8, CER = 0.125
   UpperCAmelCase__   :					Any   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.1_2_5      ) < 1e-6      )

  def 				__lowercase    (					self						:     int      ):






   '''simple docstring'''
   UpperCAmelCase__   :					str   				=   ["""werewolf"""]
   UpperCAmelCase__   :					Any   				=   ["""wereawolf"""]
   # S = 0, D = 1, I = 0, N = 8, CER = 0.125
   UpperCAmelCase__   :					List[str]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.1_2_5      ) < 1e-6      )

  def 				__lowercase    (					self						:     Dict      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[int]   				=   ["""werewolf"""]
   UpperCAmelCase__   :					int   				=   ["""wereolf"""]
   # S = 0, D = 0, I = 1, N = 8, CER = 0.125
   UpperCAmelCase__   :					Optional[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.1_2_5      ) < 1e-6      )

  def 				__lowercase    (					self						:     str      ):






   '''simple docstring'''
   UpperCAmelCase__   :					int   				=   ["""werewolf"""]
   UpperCAmelCase__   :					Dict   				=   cer.compute(predictions=A   ,references=A      )
   self.assertEqual(A   ,0.0      )

  def 				__lowercase    (					self						:     Optional[Any]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Any   				=   ["""werewolf""", """I am your father"""]
   UpperCAmelCase__   :					Optional[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertEqual(A   ,0.0      )

   UpperCAmelCase__   :					Union[str, Any]   				=   ["""werewolf""", """I am your father""", """doge"""]
   UpperCAmelCase__   :					int   				=   ["""werxwolf""", """I       am your father""", """doge"""]
   # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
   UpperCAmelCase__   :					Optional[int]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.0_3_5_7_1_4_2_8      ) < 1e-6      )

  def 				__lowercase    (					self						:     Dict      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[int]   				=   ["""My hovercraft""", """is full of eels"""]
   UpperCAmelCase__   :					Any   				=   ["""My hovercraft is full""", """ of eels"""]
   # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
   # whitespace at the front of " of eels" will be strip during preporcessing
   # so need to insert 2 whitespaces
   UpperCAmelCase__   :					Union[str, Any]   				=   cer.compute(predictions=A   ,references=A   ,concatenate_texts=A      )
   self.assertTrue(abs(char_error_rate - 0.0_7_1_4_2_8      ) < 1e-6      )

  def 				__lowercase    (					self						:     Tuple      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Dict   				=   ["""我能吞下玻璃而不伤身体"""]
   UpperCAmelCase__   :					int   				=   [""" 能吞虾玻璃而 不霜身体啦"""]
   # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
   UpperCAmelCase__   :					List[str]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5      ) < 1e-6      )

   UpperCAmelCase__   :					List[str]   				=   ["""我能吞下玻璃""", """而不伤身体"""]
   UpperCAmelCase__   :					List[Any]   				=   ["""我    能 吞 下 玻 璃""", """而不伤身体"""]
   # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
   UpperCAmelCase__   :					List[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5_4_5      ) < 1e-6      )

   UpperCAmelCase__   :					List[str]   				=   ["""我能吞下玻璃而不伤身体"""]
   UpperCAmelCase__   :					List[Any]   				=   cer.compute(predictions=A   ,references=A      )
   self.assertFalse(A   ,0.0      )




  def 				__lowercase    (					self						:     Union[str, Any]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[Any]   				=   [""""""]
   UpperCAmelCase__   :					Tuple   				=   ["""Hypothesis"""]
   with self.assertRaises(A      ):
    cer.compute(predictions=A   ,references=A      )


if __name__ == "__main__":
      unittest.main()








"""simple docstring"""




import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_reformer import ReformerTokenizer
else:
      __UpperCAmelCase	=			None


__UpperCAmelCase	=			logging.get_logger(__name__)


__UpperCAmelCase	=			'▁'

__UpperCAmelCase	=			{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase	=			{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

__UpperCAmelCase	=			{
    'google/reformer-crime-and-punishment': 52_4288,
}







class        __lowercase     (      __lowerCamelCase   ):
  snake_case_			    =    VOCAB_FILES_NAMES
  snake_case_			    =    PRETRAINED_VOCAB_FILES_MAP
  snake_case_			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_			    =    ["""input_ids""", """attention_mask"""]
  snake_case_			    =    ReformerTokenizer

  def __init__(					self						:     List[Any]   ,A						:     Dict=None   ,A						:     List[str]=None   ,A						:     Union[str, Any]="</s>"   ,A						:     List[str]="<unk>"   ,A						:     Tuple=[]   ,**A						:     Optional[int]   ,):






   '''simple docstring'''
   super().__init__(
       A   ,tokenizer_file=A   ,eos_token=A   ,unk_token=A   ,additional_special_tokens=A   ,**A   ,)

   UpperCAmelCase__   :					Tuple   				=   vocab_file
   UpperCAmelCase__   :					Optional[Any]   				=   False if not self.vocab_file else True




  def 				__lowercase    (					self						:     Any   ,A						:     str   ,A						:     Optional[str] = None      ):






   '''simple docstring'''
   if not self.can_save_slow_tokenizer:
    raise ValueError(
        """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
        """tokenizer."""      )

   if not os.path.isdir(A      ):
    logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
    return
   UpperCAmelCase__   :					Tuple   				=   os.path.join(
       A   ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

   if os.path.abspath(self.vocab_file      ) != os.path.abspath(A      ):
    copyfile(self.vocab_file   ,A      )

   return (out_vocab_file,)





