




'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


A			   =logging.get_logger(__name__)

A			   ={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A			   ={
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

A			   ={
    'openai-gpt': 5_12,
}





def 						snake_case_							(_a     :							Optional[int]					):
    UpperCAmelCase      	=		text.strip()
    if not text:
        return []
    UpperCAmelCase      	=		text.split()
    return tokens


class       _a  (     __a			):
       def __init__(     self						:     Any       ,    lowercase						:     Optional[Any]=True       ,    lowercase						:     Optional[Any]=None       ,    lowercase						:     List[Any]=True       ,    lowercase						:     Dict=None       ,    lowercase						:     Optional[int]=True       ,    ):



           '''simple docstring'''

           if never_split is None:
               UpperCAmelCase      	=		[]
           UpperCAmelCase      	=		do_lower_case
           UpperCAmelCase      	=		set(lowercase    )
           UpperCAmelCase      	=		tokenize_chinese_chars
           UpperCAmelCase      	=		strip_accents
           UpperCAmelCase      	=		do_split_on_punc
       def    A				(     self						:     str       ,    lowercase						:     Dict       ,    lowercase						:     List[Any]=None    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.never_split.union(set(lowercase    )    ) if never_split else self.never_split
           UpperCAmelCase      	=		self._clean_text(lowercase    )

           # This was added on November 1st, 2018 for the multilingual and Chinese
           # models. This is also applied to the English models now, but it doesn't
           # matter since the English models were not trained on any Chinese data
           # and generally don't have any Chinese data in them (there are Chinese
           # characters in the vocabulary because Wikipedia does have some Chinese
           # words in the English Wikipedia.).
           if self.tokenize_chinese_chars:
               UpperCAmelCase      	=		self._tokenize_chinese_chars(lowercase    )
           # prevents treating the same character with different unicode codepoints as different characters
           UpperCAmelCase      	=		unicodedata.normalize('''NFC'''       ,    lowercase    )
           UpperCAmelCase      	=		whitespace_tokenize(lowercase    )
           UpperCAmelCase      	=		[]
           for token in orig_tokens:
               if token not in never_split:
                   if self.do_lower_case:
                       UpperCAmelCase      	=		token.lower()
                       if self.strip_accents is not False:
                           UpperCAmelCase      	=		self._run_strip_accents(lowercase    )
                   elif self.strip_accents:
                       UpperCAmelCase      	=		self._run_strip_accents(lowercase    )
               split_tokens.extend(self._run_split_on_punc(lowercase       ,    lowercase    )    )

           UpperCAmelCase      	=		whitespace_tokenize(''' '''.join(lowercase    )    )
           return output_tokens
       def    A				(     self						:     Optional[int]       ,    lowercase						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		unicodedata.normalize('''NFD'''       ,    lowercase    )
           UpperCAmelCase      	=		[]
           for char in text:
               UpperCAmelCase      	=		unicodedata.category(lowercase    )
               if cat == "Mn":
                   continue
               output.append(lowercase    )
           return "".join(lowercase    )
       def    A				(     self						:     Dict       ,    lowercase						:     Dict       ,    lowercase						:     Dict=None    ):



           '''simple docstring'''

           if not self.do_split_on_punc or (never_split is not None and text in never_split):
               return [text]
           UpperCAmelCase      	=		list(lowercase    )
           UpperCAmelCase      	=		0
           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		[]
           while i < len(lowercase    ):
               UpperCAmelCase      	=		chars[i]
               if _is_punctuation(lowercase    ):
                   output.append([char]    )
                   UpperCAmelCase      	=		True
               else:
                   if start_new_word:
                       output.append([]    )
                   UpperCAmelCase      	=		False
                   output[-1].append(lowercase    )
               i += 1

           return ["".join(lowercase    ) for x in output]
       def    A				(     self						:     List[str]       ,    lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for char in text:
               UpperCAmelCase      	=		ord(lowercase    )
               if self._is_chinese_char(lowercase    ):
                   output.append(''' '''    )
                   output.append(lowercase    )
                   output.append(''' '''    )
               else:
                   output.append(lowercase    )
           return "".join(lowercase    )
       def    A				(     self						:     int       ,    lowercase						:     Union[str, Any]    ):



           '''simple docstring'''

           if (
               (cp >= 0X4e_00 and cp <= 0X9f_ff)
               or (cp >= 0X34_00 and cp <= 0X4d_bf)  #
               or (cp >= 0X2_00_00 and cp <= 0X2_a6_df)  #
               or (cp >= 0X2_a7_00 and cp <= 0X2_b7_3f)  #
               or (cp >= 0X2_b7_40 and cp <= 0X2_b8_1f)  #
               or (cp >= 0X2_b8_20 and cp <= 0X2_ce_af)  #
               or (cp >= 0Xf9_00 and cp <= 0Xfa_ff)
               or (cp >= 0X2_f8_00 and cp <= 0X2_fa_1f)  #
           ):  #
               return True

           return False







       def    A				(     self						:     Optional[int]       ,    lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for char in text:
               UpperCAmelCase      	=		ord(lowercase    )
               if cp == 0 or cp == 0Xff_fd or _is_control(lowercase    ):
                   continue
               if _is_whitespace(lowercase    ):
                   output.append(''' '''    )
               else:
                   output.append(lowercase    )
           return "".join(lowercase    )






def 						snake_case_							(_a     :							Tuple					):
    UpperCAmelCase      	=		set()
    UpperCAmelCase      	=		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)					)
        UpperCAmelCase      	=		char
    return pairs





def 						snake_case_							(_a     :							Dict					):
    UpperCAmelCase      	=		text.replace('''—'''   ,       '''-'''					)
    UpperCAmelCase      	=		text.replace('''–'''   ,       '''-'''					)
    UpperCAmelCase      	=		text.replace('''―'''   ,       '''-'''					)
    UpperCAmelCase      	=		text.replace('''…'''   ,       '''...'''					)
    UpperCAmelCase      	=		text.replace('''´'''   ,       '''\''''					)
    UpperCAmelCase      	=		re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''   ,       R''' \1 '''   ,       _a					)
    UpperCAmelCase      	=		re.sub(R'''\s*\n\s*'''   ,       ''' \n '''   ,       _a					)
    UpperCAmelCase      	=		re.sub(R'''[^\S\n]+'''   ,       ''' '''   ,       _a					)
    return text.strip()







class       _a  (     __a			):
       __a    :					Union[str, Any]           = VOCAB_FILES_NAMES
       __a    :					Union[str, Any]           = PRETRAINED_VOCAB_FILES_MAP
       __a    :					Optional[Any]           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __a    :					Optional[Any]           = ["""input_ids""", """attention_mask"""]
       def __init__(     self						:     Optional[int]       ,    lowercase						:     int       ,    lowercase						:     List[str]       ,    lowercase						:     Optional[int]="<unk>"       ,    **lowercase						:     Optional[Any]    ):



           '''simple docstring'''

           super().__init__(unk_token=lowercase       ,    **lowercase    )

           try:
               import ftfy
               from spacy.lang.en import English

               UpperCAmelCase      	=		English()
               UpperCAmelCase      	=		_nlp.tokenizer
               UpperCAmelCase      	=		ftfy.fix_text
           except ImportError:
               logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''    )
               UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )
               UpperCAmelCase      	=		None

           with open(lowercase       ,    encoding='''utf-8'''    ) as vocab_handle:
               UpperCAmelCase      	=		json.load(lowercase    )
           UpperCAmelCase      	=		{v: k for k, v in self.encoder.items()}
           with open(lowercase       ,    encoding='''utf-8'''    ) as merges_handle:
               UpperCAmelCase      	=		merges_handle.read().split('''\n'''    )[1:-1]
           UpperCAmelCase      	=		[tuple(merge.split()    ) for merge in merges]
           UpperCAmelCase      	=		dict(zip(lowercase       ,    range(len(lowercase    )    )    )    )
           UpperCAmelCase      	=		{}
       @property
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           return True
       @property
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           return len(self.encoder    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           return dict(self.encoder       ,    **self.added_tokens_encoder    )
       def    A				(     self						:     Union[str, Any]       ,    lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		tuple(token[:-1]    ) + (token[-1] + '''</w>''',)
           if token in self.cache:
               return self.cache[token]
           UpperCAmelCase      	=		get_pairs(lowercase    )

           if not pairs:
               return token + "</w>"

           while True:
               UpperCAmelCase      	=		min(lowercase       ,    key=lambda lowercase    : self.bpe_ranks.get(lowercase       ,    float('''inf'''    )    )    )
               if bigram not in self.bpe_ranks:
                   break
               UpperCAmelCase	,					UpperCAmelCase      	=		bigram
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		0
               while i < len(lowercase    ):
                   try:
                       UpperCAmelCase      	=		word.index(lowercase       ,    lowercase    )
                   except ValueError:
                       new_word.extend(word[i:]    )
                       break
                   else:
                       new_word.extend(word[i:j]    )
                       UpperCAmelCase      	=		j

                   if word[i] == first and i < len(lowercase    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                   else:
                       new_word.append(word[i]    )
                       i += 1
               UpperCAmelCase      	=		tuple(lowercase    )
               UpperCAmelCase      	=		new_word
               if len(lowercase    ) == 1:
                   break
               else:
                   UpperCAmelCase      	=		get_pairs(lowercase    )
           UpperCAmelCase      	=		''' '''.join(lowercase    )
           if word == "\n  </w>":
               UpperCAmelCase      	=		'''\n</w>'''
           UpperCAmelCase      	=		word
           return word
       def    A				(     self						:     List[Any]       ,    lowercase						:     List[str]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           if self.fix_text is None:
               # Using BERT's BasicTokenizer
               UpperCAmelCase      	=		self.nlp.tokenize(lowercase    )
               for token in text:
                   split_tokens.extend(list(self.bpe(lowercase    ).split(''' '''    )    )    )
           else:
               # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
               UpperCAmelCase      	=		self.nlp(text_standardize(self.fix_text(lowercase    )    )    )
               for token in text:
                   split_tokens.extend(list(self.bpe(token.text.lower()    ).split(''' '''    )    )    )
           return split_tokens
       def    A				(     self						:     Union[str, Any]       ,    lowercase						:     str    ):



           '''simple docstring'''

           return self.encoder.get(lowercase       ,    self.encoder.get(self.unk_token    )    )
       def    A				(     self						:     Union[str, Any]       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           return self.decoder.get(lowercase       ,    self.unk_token    )
       def    A				(     self						:     Optional[int]       ,    lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		''''''.join(lowercase    ).replace('''</w>'''       ,    ''' '''    ).strip()
           return out_string







       def    A				(     self						:     Tuple       ,    lowercase						:     str       ,    lowercase						:     Optional[str] = None    ):



           '''simple docstring'''

           if not os.path.isdir(lowercase    ):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
               return
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as f:
               f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=lowercase       ,    ensure_ascii=lowercase    ) + '''\n'''    )

           UpperCAmelCase      	=		0
           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as writer:
               writer.write('''#version: 0.2\n'''    )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda lowercase    : kv[1]    ):
                   if index != token_index:
                       logger.warning(
                           f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                           ''' Please check that the tokenizer is not corrupted!'''    )
                       UpperCAmelCase      	=		token_index
                   writer.write(''' '''.join(lowercase    ) + '''\n'''    )
                   index += 1

           return vocab_file, merge_file










'''simple docstring'''







import unittest

from transformers import is_torch_available, load_tool

from .test_tools_common import ToolTesterMixin


if is_torch_available():
   import torch


class       _a  (     unittest.TestCase							,							__a			):
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		load_tool('''speech-to-text'''    )
           self.tool.setup()
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tool(torch.ones(3_000    )    )
           self.assertEqual(lowercase       ,    ''' you'''    )







       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tool(audio=torch.ones(3_000    )    )
           self.assertEqual(lowercase       ,    ''' you'''    )




