

'''simple docstring'''






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


snake_case_					  =	logging.get_logger(__name__)

snake_case_					  =	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

snake_case_					  =	{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

snake_case_					  =	{
    'openai-gpt': 5_1_2,
}


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     Tuple   )							->       Dict:
      SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	text.strip()
      if not text:
            return []
      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	text.split()
      return tokens
class      SCREAMING_SNAKE_CASE__      (    lowerCAmelCase__				):




    def __init__( self    ,					lowercase__=True    ,					lowercase__=None    ,					lowercase__=True    ,					lowercase__=None    ,					lowercase__=True    ,					):






          """simple docstring"""






          if never_split is None:
                SCREAMING_SNAKE_CASE_				:      int					=	[]
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	do_lower_case
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	set(_A      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenize_chinese_chars
          SCREAMING_SNAKE_CASE_				:      str					=	strip_accents
          SCREAMING_SNAKE_CASE_				:      Dict					=	do_split_on_punc




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__=None      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	self.never_split.union(set(_A      )      ) if never_split else self.never_split
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	self._clean_text(_A      )

          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
                SCREAMING_SNAKE_CASE_				:      Dict					=	self._tokenize_chinese_chars(_A      )
          # prevents treating the same character with different unicode codepoints as different characters
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	unicodedata.normalize("NFC"    ,					_A      )
          SCREAMING_SNAKE_CASE_				:      List[str]					=	whitespace_tokenize(_A      )
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	[]
          for token in orig_tokens:
                if token not in never_split:
                      if self.do_lower_case:
                            SCREAMING_SNAKE_CASE_				:      Dict					=	token.lower()
                            if self.strip_accents is not False:
                                  SCREAMING_SNAKE_CASE_				:      Optional[int]					=	self._run_strip_accents(_A      )
                      elif self.strip_accents:
                            SCREAMING_SNAKE_CASE_				:      Dict					=	self._run_strip_accents(_A      )
                split_tokens.extend(self._run_split_on_punc(_A    ,					_A      )      )

          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	whitespace_tokenize(" ".join(_A      )      )
          return output_tokens




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Any					=	unicodedata.normalize("NFD"    ,					_A      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	[]
          for char in text:
                SCREAMING_SNAKE_CASE_				:      Optional[int]					=	unicodedata.category(_A      )
                if cat == "Mn":
                      continue
                output.append(_A      )
          return "".join(_A      )




    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__=None      ):






          """simple docstring"""






          if not self.do_split_on_punc or (never_split is not None and text in never_split):
                return [text]
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	list(_A      )
          SCREAMING_SNAKE_CASE_				:      Any					=	0
          SCREAMING_SNAKE_CASE_				:      str					=	True
          SCREAMING_SNAKE_CASE_				:      int					=	[]
          while i < len(_A      ):
                SCREAMING_SNAKE_CASE_				:      Tuple					=	chars[i]
                if _is_punctuation(_A      ):
                      output.append([char]      )
                      SCREAMING_SNAKE_CASE_				:      str					=	True
                else:
                      if start_new_word:
                            output.append([]      )
                      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	False
                      output[-1].append(_A      )
                i += 1

          return ["".join(_A      ) for x in output]




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	[]
          for char in text:
                SCREAMING_SNAKE_CASE_				:      List[Any]					=	ord(_A      )
                if self._is_chinese_char(_A      ):
                      output.append(" "      )
                      output.append(_A      )
                      output.append(" "      )
                else:
                      output.append(_A      )
          return "".join(_A      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          if (
              (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
              or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
              or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
              or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
              or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
              or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
              or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
              or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
          ):  #
                return True

          return False







    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Dict					=	[]
          for char in text:
                SCREAMING_SNAKE_CASE_				:      Tuple					=	ord(_A      )
                if cp == 0 or cp == 0Xf_f_f_d or _is_control(_A      ):
                      continue
                if _is_whitespace(_A      ):
                      output.append(" "      )
                else:
                      output.append(_A      )
          return "".join(_A      )






def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     Union[str, Any]   )							->       Union[str, Any]:
      SCREAMING_SNAKE_CASE_				:      str					=	set()
      SCREAMING_SNAKE_CASE_				:      str					=	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)   )
            SCREAMING_SNAKE_CASE_				:      Tuple					=	char
      return pairs


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     List[str]   )							->       Dict:
      SCREAMING_SNAKE_CASE_				:      Optional[int]					=	text.replace("—"  ,  "-"   )
      SCREAMING_SNAKE_CASE_				:      Tuple					=	text.replace("–"  ,  "-"   )
      SCREAMING_SNAKE_CASE_				:      Any					=	text.replace("―"  ,  "-"   )
      SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	text.replace("…"  ,  "..."   )
      SCREAMING_SNAKE_CASE_				:      List[Any]					=	text.replace("´"  ,  "\'"   )
      SCREAMING_SNAKE_CASE_				:      Any					=	re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"  ,  R" \1 "  ,  SCREAMING_SNAKE_CASE_   )
      SCREAMING_SNAKE_CASE_				:      Optional[int]					=	re.sub(R"\s*\n\s*"  ,  " \n "  ,  SCREAMING_SNAKE_CASE_   )
      SCREAMING_SNAKE_CASE_				:      List[Any]					=	re.sub(R"[^\S\n]+"  ,  " "  ,  SCREAMING_SNAKE_CASE_   )
      return text.strip()





class      SCREAMING_SNAKE_CASE__      (    lowerCAmelCase__				):
    _A         =							VOCAB_FILES_NAMES
    _A         =							PRETRAINED_VOCAB_FILES_MAP
    _A         =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A         =							["input_ids", "attention_mask"]




    def __init__( self    ,					lowercase__    ,					lowercase__    ,					lowercase__="<unk>"    ,					**lowercase__      ):






          """simple docstring"""






          super().__init__(unk_token=_A    ,					**_A      )

          try:
                import ftfy
                from spacy.lang.en import English

                SCREAMING_SNAKE_CASE_				:      str					=	English()
                SCREAMING_SNAKE_CASE_				:      Any					=	_nlp.tokenizer
                SCREAMING_SNAKE_CASE_				:      List[str]					=	ftfy.fix_text
          except ImportError:
                logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."      )
                SCREAMING_SNAKE_CASE_				:      int					=	BasicTokenizer(do_lower_case=_A      )
                SCREAMING_SNAKE_CASE_				:      int					=	None

          with open(_A    ,					encoding="utf-8"      ) as vocab_handle:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	json.load(_A      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	{v: k for k, v in self.encoder.items()}
          with open(_A    ,					encoding="utf-8"      ) as merges_handle:
                SCREAMING_SNAKE_CASE_				:      Any					=	merges_handle.read().split("\n"      )[1:-1]
          SCREAMING_SNAKE_CASE_				:      Tuple					=	[tuple(merge.split()      ) for merge in merges]
          SCREAMING_SNAKE_CASE_				:      Dict					=	dict(zip(_A    ,					range(len(_A      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	{}




    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return True




    @property
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return len(self.encoder      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          return dict(self.encoder    ,					**self.added_tokens_encoder      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	tuple(token[:-1]      ) + (token[-1] + '''</w>''',)
          if token in self.cache:
                return self.cache[token]
          SCREAMING_SNAKE_CASE_				:      Any					=	get_pairs(_A      )

          if not pairs:
                return token + "</w>"

          while True:
                SCREAMING_SNAKE_CASE_				:      Dict					=	min(_A    ,					key=lambda lowercase__      : self.bpe_ranks.get(_A    ,					float("inf"      )      )      )
                if bigram not in self.bpe_ranks:
                      break
                SCREAMING_SNAKE_CASE_				:      str					=	bigram
                SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	[]
                SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	0
                while i < len(_A      ):
                      try:
                            SCREAMING_SNAKE_CASE_				:      str					=	word.index(_A    ,					_A      )
                      except ValueError:
                            new_word.extend(word[i:]      )
                            break
                      else:
                            new_word.extend(word[i:j]      )
                            SCREAMING_SNAKE_CASE_				:      List[Any]					=	j

                      if word[i] == first and i < len(_A      ) - 1 and word[i + 1] == second:
                            new_word.append(first + second      )
                            i += 2
                      else:
                            new_word.append(word[i]      )
                            i += 1
                SCREAMING_SNAKE_CASE_				:      str					=	tuple(_A      )
                SCREAMING_SNAKE_CASE_				:      str					=	new_word
                if len(_A      ) == 1:
                      break
                else:
                      SCREAMING_SNAKE_CASE_				:      Any					=	get_pairs(_A      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	''' '''.join(_A      )
          if word == "\n  </w>":
                SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	'''\n</w>'''
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	word
          return word




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Tuple					=	[]
          if self.fix_text is None:
                # Using BERT's BasicTokenizer
                SCREAMING_SNAKE_CASE_				:      List[str]					=	self.nlp.tokenize(_A      )
                for token in text:
                      split_tokens.extend(list(self.bpe(_A      ).split(" "      )      )      )
          else:
                # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.nlp(text_standardize(self.fix_text(_A      )      )      )
                for token in text:
                      split_tokens.extend(list(self.bpe(token.text.lower()      ).split(" "      )      )      )
          return split_tokens




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          return self.encoder.get(_A    ,					self.encoder.get(self.unk_token      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          return self.decoder.get(_A    ,					self.unk_token      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	''''''.join(_A      ).replace("</w>"    ,					" "      ).strip()
          return out_string







    def   __lowerCamelCase				( self    ,					lowercase__    ,					lowercase__ = None      ):






          """simple docstring"""






          if not os.path.isdir(_A      ):
                logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
                return
          SCREAMING_SNAKE_CASE_				:      str					=	os.path.join(
              _A    ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      Tuple					=	os.path.join(
              _A    ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

          with open(_A    ,					"w"    ,					encoding="utf-8"      ) as f:
                f.write(json.dumps(self.encoder    ,					indent=2    ,					sort_keys=_A    ,					ensure_ascii=_A      ) + "\n"      )

          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	0
          with open(_A    ,					"w"    ,					encoding="utf-8"      ) as writer:
                writer.write("#version: 0.2\n"      )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,					key=lambda lowercase__      : kv[1]      ):
                      if index != token_index:
                            logger.warning(
                                F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                " Please check that the tokenizer is not corrupted!"      )
                            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	token_index
                      writer.write(" ".join(_A      ) + "\n"      )
                      index += 1

          return vocab_file, merge_file




'''simple docstring'''






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							HerbertTokenizer
    _A         =							HerbertTokenizerFast
    _A         =							True




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"    ,					encoding="utf-8"      ) as f_data:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	f_data.read().replace("\n\n"    ,					"\n"      ).strip()

          SCREAMING_SNAKE_CASE_				:      Dict					=	[
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	dict(zip(lowercase__    ,					range(len(lowercase__      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

          SCREAMING_SNAKE_CASE_				:      Any					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["merges_file"]      )
          with open(self.vocab_file    ,					"w"      ) as fp:
                fp.write(json.dumps(lowercase__      )      )
          with open(self.merges_file    ,					"w"      ) as fp:
                fp.write("\n".join(lowercase__      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	"lower newer"
          SCREAMING_SNAKE_CASE_				:      str					=	"lower newer"
          return input_text, output_text




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.tokenizer_class(vocab_file=self.vocab_file    ,					merges_file=self.merges_file      )

          SCREAMING_SNAKE_CASE_				:      int					=	"lower"
          SCREAMING_SNAKE_CASE_				:      List[str]					=	["low", "er</w>"]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokens + ["<unk>"]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__      )    ,					lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if not self.test_rust_tokenizer:
                return

          SCREAMING_SNAKE_CASE_				:      Dict					=	self.get_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	"lower,newer"

          SCREAMING_SNAKE_CASE_				:      int					=	tokenizer.tokenize(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	rust_tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	rust_tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.encode(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	rust_tokenizer.encode(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )







    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode("konstruowanie sekwencji"    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	tokenizer.encode("konstruowanie wielu sekwencji"    ,					add_special_tokens=lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.build_inputs_with_special_tokens(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.build_inputs_with_special_tokens(lowercase__    ,					lowercase__      )

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]

