
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


SCREAMING_SNAKE_CASE_			      =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_			      =					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

SCREAMING_SNAKE_CASE_			      =					{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

SCREAMING_SNAKE_CASE_			      =					{
    '''openai-gpt''': 512,
}

def 				lowercase      (_lowerCAmelCase		):
						__lowerCAmelCase        =       text.strip()
						if not text:
												return []
						__lowerCAmelCase        =       text.split()
						return tokens
class      lowerCAmelCase_						(							A__				):



							'''simple docstring'''





							def __init__(				self  ,							snake_case_=True  ,							snake_case_=None  ,							snake_case_=True  ,							snake_case_=None  ,							snake_case_=True  ,							)   ->       Tuple:
													if never_split is None:
																			__lowerCAmelCase        =       []
													__lowerCAmelCase        =       do_lower_case
													__lowerCAmelCase        =       set(snake_case_							)
													__lowerCAmelCase        =       tokenize_chinese_chars
													__lowerCAmelCase        =       strip_accents
													__lowerCAmelCase        =       do_split_on_punc




							def  A__		(				self  ,							snake_case_  ,							snake_case_=None							)   ->       Tuple:
													__lowerCAmelCase        =       self.never_split.union(set(snake_case_							)							) if never_split else self.never_split
													__lowerCAmelCase        =       self._clean_text(snake_case_							)

													# This was added on November 1st, 2018 for the multilingual and Chinese
													# models. This is also applied to the English models now, but it doesn't
													# matter since the English models were not trained on any Chinese data
													# and generally don't have any Chinese data in them (there are Chinese
													# characters in the vocabulary because Wikipedia does have some Chinese
													# words in the English Wikipedia.).
													if self.tokenize_chinese_chars:
																			__lowerCAmelCase        =       self._tokenize_chinese_chars(snake_case_							)
													# prevents treating the same character with different unicode codepoints as different characters
													__lowerCAmelCase        =       unicodedata.normalize("""NFC"""  ,							snake_case_							)
													__lowerCAmelCase        =       whitespace_tokenize(snake_case_							)
													__lowerCAmelCase        =       []
													for token in orig_tokens:
																			if token not in never_split:
																									if self.do_lower_case:
																															__lowerCAmelCase        =       token.lower()
																															if self.strip_accents is not False:
																																					__lowerCAmelCase        =       self._run_strip_accents(snake_case_							)
																									elif self.strip_accents:
																															__lowerCAmelCase        =       self._run_strip_accents(snake_case_							)
																			split_tokens.extend(self._run_split_on_punc(snake_case_  ,							snake_case_							)							)

													__lowerCAmelCase        =       whitespace_tokenize(""" """.join(snake_case_							)							)
													return output_tokens




							def  A__		(				self  ,							snake_case_							)   ->       List[Any]:
													__lowerCAmelCase        =       unicodedata.normalize("""NFD"""  ,							snake_case_							)
													__lowerCAmelCase        =       []
													for char in text:
																			__lowerCAmelCase        =       unicodedata.category(snake_case_							)
																			if cat == "Mn":
																									continue
																			output.append(snake_case_							)
													return "".join(snake_case_							)




							def  A__		(				self  ,							snake_case_  ,							snake_case_=None							)   ->       List[Any]:
													if not self.do_split_on_punc or (never_split is not None and text in never_split):
																			return [text]
													__lowerCAmelCase        =       list(snake_case_							)
													__lowerCAmelCase        =       0
													__lowerCAmelCase        =       True
													__lowerCAmelCase        =       []
													while i < len(snake_case_							):
																			__lowerCAmelCase        =       chars[i]
																			if _is_punctuation(snake_case_							):
																									output.append([char]							)
																									__lowerCAmelCase        =       True
																			else:
																									if start_new_word:
																															output.append([]							)
																									__lowerCAmelCase        =       False
																									output[-1].append(snake_case_							)
																			i += 1

													return ["".join(snake_case_							) for x in output]




							def  A__		(				self  ,							snake_case_							)   ->       str:
													__lowerCAmelCase        =       []
													for char in text:
																			__lowerCAmelCase        =       ord(snake_case_							)
																			if self._is_chinese_char(snake_case_							):
																									output.append(""" """							)
																									output.append(snake_case_							)
																									output.append(""" """							)
																			else:
																									output.append(snake_case_							)
													return "".join(snake_case_							)




							def  A__		(				self  ,							snake_case_							)   ->       Union[str, Any]:
													# This defines a "chinese character" as anything in the CJK Unicode block:
													#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
													#
													# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
													# despite its name. The modern Korean Hangul alphabet is a different block,
													# as is Japanese Hiragana and Katakana. Those alphabets are used to write
													# space-separated words, so they are not treated specially and handled
													# like the all of the other languages.
													if (
													    (cp >= 0x4e00 and cp <= 0x9fff)
													    or (cp >= 0x3400 and cp <= 0x4dbf)  #
													    or (cp >= 0x20000 and cp <= 0x2a6df)  #
													    or (cp >= 0x2a700 and cp <= 0x2b73f)  #
													    or (cp >= 0x2b740 and cp <= 0x2b81f)  #
													    or (cp >= 0x2b820 and cp <= 0x2ceaf)  #
													    or (cp >= 0xf900 and cp <= 0xfaff)
													    or (cp >= 0x2f800 and cp <= 0x2fa1f)  #
													):  #
																			return True

													return False




							def  A__		(				self  ,							snake_case_							)   ->       Union[str, Any]:
													__lowerCAmelCase        =       []
													for char in text:
																			__lowerCAmelCase        =       ord(snake_case_							)
																			if cp == 0 or cp == 0xfffd or _is_control(snake_case_							):
																									continue
																			if _is_whitespace(snake_case_							):
																									output.append(""" """							)
																			else:
																									output.append(snake_case_							)
													return "".join(snake_case_							)

def 				lowercase      (_lowerCAmelCase		):
						__lowerCAmelCase        =       set()
						__lowerCAmelCase        =       word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)		)
												__lowerCAmelCase        =       char
						return pairs

def 				lowercase      (_lowerCAmelCase		):
						__lowerCAmelCase        =       text.replace("""—"""				,  """-"""		)
						__lowerCAmelCase        =       text.replace("""–"""				,  """-"""		)
						__lowerCAmelCase        =       text.replace("""―"""				,  """-"""		)
						__lowerCAmelCase        =       text.replace("""…"""				,  """..."""		)
						__lowerCAmelCase        =       text.replace("""´"""				,  """'"""		)
						__lowerCAmelCase        =       re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""				,  r""" \1 """				,  _lowerCAmelCase		)
						__lowerCAmelCase        =       re.sub(r"""\s*\n\s*"""				,  """ \n """				,  _lowerCAmelCase		)
						__lowerCAmelCase        =       re.sub(r"""[^\S\n]+"""				,  """ """				,  _lowerCAmelCase		)
						return text.strip()



class      lowerCAmelCase_						(							A__				):



							'''simple docstring'''





							_snake_case		     =						VOCAB_FILES_NAMES
							_snake_case		     =						PRETRAINED_VOCAB_FILES_MAP
							_snake_case		     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_snake_case		     =						['''input_ids''', '''attention_mask''']




							def __init__(				self  ,							snake_case_  ,							snake_case_  ,							snake_case_="<unk>"  ,							**snake_case_							)   ->       List[str]:
													super().__init__(unk_token=snake_case_  ,							**snake_case_							)

													try:
																			import ftfy
																			from spacy.lang.en import English

																			__lowerCAmelCase        =       English()
																			__lowerCAmelCase        =       _nlp.tokenizer
																			__lowerCAmelCase        =       ftfy.fix_text
													except ImportError:
																			logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""							)
																			__lowerCAmelCase        =       BasicTokenizer(do_lower_case=snake_case_							)
																			__lowerCAmelCase        =       None

													with open(snake_case_  ,							encoding="""utf-8"""							) as vocab_handle:
																			__lowerCAmelCase        =       json.load(snake_case_							)
													__lowerCAmelCase        =       {v: k for k, v in self.encoder.items()}
													with open(snake_case_  ,							encoding="""utf-8"""							) as merges_handle:
																			__lowerCAmelCase        =       merges_handle.read().split("""\n"""							)[1:-1]
													__lowerCAmelCase        =       [tuple(merge.split()							) for merge in merges]
													__lowerCAmelCase        =       dict(zip(snake_case_  ,							range(len(snake_case_							)							)							)							)
													__lowerCAmelCase        =       {}




							@property
							def  A__		(				self							)   ->       List[str]:
													return True




							@property
							def  A__		(				self							)   ->       List[str]:
													return len(self.encoder							)




							def  A__		(				self							)   ->       List[Any]:
													return dict(self.encoder  ,							**self.added_tokens_encoder							)




							def  A__		(				self  ,							snake_case_							)   ->       Optional[int]:
													__lowerCAmelCase        =       tuple(token[:-1]							) + (token[-1] + """</w>""",)
													if token in self.cache:
																			return self.cache[token]
													__lowerCAmelCase        =       get_pairs(snake_case_							)

													if not pairs:
																			return token + "</w>"

													while True:
																			__lowerCAmelCase        =       min(snake_case_  ,							key=lambda snake_case_							: self.bpe_ranks.get(snake_case_  ,							float("""inf"""							)							)							)
																			if bigram not in self.bpe_ranks:
																									break
																			__lowerCAmelCase ,      __lowerCAmelCase        =       bigram
																			__lowerCAmelCase        =       []
																			__lowerCAmelCase        =       0
																			while i < len(snake_case_							):
																									try:
																															__lowerCAmelCase        =       word.index(snake_case_  ,							snake_case_							)
																									except ValueError:
																															new_word.extend(word[i:]							)
																															break
																									else:
																															new_word.extend(word[i:j]							)
																															__lowerCAmelCase        =       j

																									if word[i] == first and i < len(snake_case_							) - 1 and word[i + 1] == second:
																															new_word.append(first + second							)
																															i += 2
																									else:
																															new_word.append(word[i]							)
																															i += 1
																			__lowerCAmelCase        =       tuple(snake_case_							)
																			__lowerCAmelCase        =       new_word
																			if len(snake_case_							) == 1:
																									break
																			else:
																									__lowerCAmelCase        =       get_pairs(snake_case_							)
													__lowerCAmelCase        =       """ """.join(snake_case_							)
													if word == "\n  </w>":
																			__lowerCAmelCase        =       """\n</w>"""
													__lowerCAmelCase        =       word
													return word




							def  A__		(				self  ,							snake_case_							)   ->       Any:
													__lowerCAmelCase        =       []
													if self.fix_text is None:
																			# Using BERT's BasicTokenizer
																			__lowerCAmelCase        =       self.nlp.tokenize(snake_case_							)
																			for token in text:
																									split_tokens.extend(list(self.bpe(snake_case_							).split(""" """							)							)							)
													else:
																			# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																			__lowerCAmelCase        =       self.nlp(text_standardize(self.fix_text(snake_case_							)							)							)
																			for token in text:
																									split_tokens.extend(list(self.bpe(token.text.lower()							).split(""" """							)							)							)
													return split_tokens




							def  A__		(				self  ,							snake_case_							)   ->       Union[str, Any]:
													return self.encoder.get(snake_case_  ,							self.encoder.get(self.unk_token							)							)




							def  A__		(				self  ,							snake_case_							)   ->       Optional[Any]:
													return self.decoder.get(snake_case_  ,							self.unk_token							)




							def  A__		(				self  ,							snake_case_							)   ->       Optional[int]:
													__lowerCAmelCase        =       """""".join(snake_case_							).replace("""</w>"""  ,							""" """							).strip()
													return out_string




							def  A__		(				self  ,							snake_case_  ,							snake_case_ = None							)   ->       Tuple[str]:
													if not os.path.isdir(snake_case_							):
																			logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
																			return
													__lowerCAmelCase        =       os.path.join(
													    snake_case_  ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
													__lowerCAmelCase        =       os.path.join(
													    snake_case_  ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

													with open(snake_case_  ,							"""w"""  ,							encoding="""utf-8"""							) as f:
																			f.write(json.dumps(self.encoder  ,							indent=2  ,							sort_keys=snake_case_  ,							ensure_ascii=snake_case_							) + """\n"""							)

													__lowerCAmelCase        =       0
													with open(snake_case_  ,							"""w"""  ,							encoding="""utf-8"""							) as writer:
																			writer.write("""#version: 0.2\n"""							)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,							key=lambda snake_case_							: kv[1]							):
																									if index != token_index:
																															logger.warning(
																															    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																															    """ Please check that the tokenizer is not corrupted!"""							)
																															__lowerCAmelCase        =       token_index
																									writer.write(""" """.join(snake_case_							) + """\n"""							)
																									index += 1

													return vocab_file, merge_file


"""simple docstring"""
import unittest

from transformers import is_torch_available, load_tool

from .test_tools_common import ToolTesterMixin


if is_torch_available():
			import torch
class      lowerCAmelCase_						(							unittest.TestCase							,							A__				):



							'''simple docstring'''





							def  A__		(				self							)   ->       Any:
													__lowerCAmelCase        =       load_tool("""speech-to-text"""							)
													self.tool.setup()




							def  A__		(				self							)   ->       int:
													__lowerCAmelCase        =       self.tool(torch.ones(3_000							)							)
													self.assertEqual(snake_case_  ,							""" you"""							)




							def  A__		(				self							)   ->       Optional[int]:
													__lowerCAmelCase        =       self.tool(audio=torch.ones(3_000							)							)
													self.assertEqual(snake_case_  ,							""" you"""							)
