


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


lowerCAmelCase__						=			logging.get_logger(__name__)

lowerCAmelCase__						=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase__						=			{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

lowerCAmelCase__						=			{
    'Salesforce/codegen-350M-mono': 20_48,
}
@lru_cache()
def 							_UpperCAmelCase    ():
       _A				:	List[str]       =			(
           list(range(ord("!"						)    ,							ord("~"						) + 1						)						) + list(range(ord("¡"						)    ,							ord("¬"						) + 1						)						) + list(range(ord("®"						)    ,							ord("ÿ"						) + 1						)						)
       )
       _A				:	Dict       =			bs[:]
       _A				:	List[str]       =			0
       for b in range(2**8						):
              if b not in bs:
                     bs.append(UpperCamelCase__						)
                     cs.append(2**8 + n						)
                     n += 1
       _A				:	str       =			[chr(UpperCamelCase__						) for n in cs]
       return dict(zip(UpperCamelCase__    ,							UpperCamelCase__						)						)
def 							_UpperCAmelCase    (UpperCamelCase__					:  str						):
       _A				:	Optional[Any]       =			set()
       _A				:	Any       =			word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)						)
              _A				:	Dict       =			char
       return pairs







class 					lowerCAmelCase__	(    a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE        =			["input_ids", "attention_mask"]


     def __init__(						self			,						__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase="replace"			,						__lowerCamelCase="<|endoftext|>"			,						__lowerCamelCase="<|endoftext|>"			,						__lowerCamelCase="<|endoftext|>"			,						__lowerCamelCase=None			,						__lowerCamelCase=False			,						__lowerCamelCase=False			,						**__lowerCamelCase			,						)      ->    Union[str, Any]:
            _A				:	Optional[int]       =			AddedToken(__lowerCamelCase			,						lstrip=__lowerCamelCase			,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase			,						__lowerCamelCase) else bos_token
            _A				:	List[Any]       =			AddedToken(__lowerCamelCase			,						lstrip=__lowerCamelCase			,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase			,						__lowerCamelCase) else eos_token
            _A				:	List[str]       =			AddedToken(__lowerCamelCase			,						lstrip=__lowerCamelCase			,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase			,						__lowerCamelCase) else unk_token
            _A				:	Any       =			AddedToken(__lowerCamelCase			,						lstrip=__lowerCamelCase			,						rstrip=__lowerCamelCase) if isinstance(__lowerCamelCase			,						__lowerCamelCase) else pad_token
            super().__init__(
                errors=__lowerCamelCase			,						unk_token=__lowerCamelCase			,						bos_token=__lowerCamelCase			,						eos_token=__lowerCamelCase			,						pad_token=__lowerCamelCase			,						add_prefix_space=__lowerCamelCase			,						add_bos_token=__lowerCamelCase			,						**__lowerCamelCase			,						)
            _A				:	int       =			add_bos_token

            with open(__lowerCamelCase			,						encoding="utf-8") as vocab_handle:
                   _A				:	Optional[int]       =			json.load(__lowerCamelCase)
            _A				:	Dict       =			{v: k for k, v in self.encoder.items()}
            _A				:	Dict       =			errors  # how to handle errors in decoding
            _A				:	int       =			bytes_to_unicode()
            _A				:	str       =			{v: k for k, v in self.byte_encoder.items()}
            with open(__lowerCamelCase			,						encoding="utf-8") as merges_handle:
                   _A				:	Dict       =			merges_handle.read().split("\n")[1:-1]
            _A				:	int       =			[tuple(merge.split()) for merge in bpe_merges]
            _A				:	Dict       =			dict(zip(__lowerCamelCase			,						range(len(__lowerCamelCase))))
            _A				:	Union[str, Any]       =			{}
            _A				:	Optional[int]       =			add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            _A				:	str       =			re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")


     @property
     def 			_lowerCamelCase			(						self)      ->    Any:
            return len(self.encoder)


     def 			_lowerCamelCase			(						self)      ->    Any:
            return dict(self.encoder			,						**self.added_tokens_encoder)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    Optional[Any]:
            if token in self.cache:
                   return self.cache[token]
            _A				:	str       =			tuple(__lowerCamelCase)
            _A				:	Union[str, Any]       =			get_pairs(__lowerCamelCase)

            if not pairs:
                   return token

            while True:
                   _A				:	List[Any]       =			min(__lowerCamelCase			,						key=lambda __lowerCamelCase: self.bpe_ranks.get(__lowerCamelCase			,						float("inf")))
                   if bigram not in self.bpe_ranks:
                          break
                   _A				:	List[str]       =			bigram
                   _A				:	List[Any]       =			[]
                   _A				:	Optional[int]       =			0
                   while i < len(__lowerCamelCase):
                          try:
                                 _A				:	Dict       =			word.index(__lowerCamelCase			,						__lowerCamelCase)
                          except ValueError:
                                 new_word.extend(word[i:])
                                 break
                          else:
                                 new_word.extend(word[i:j])
                                 _A				:	Tuple       =			j

                          if word[i] == first and i < len(__lowerCamelCase) - 1 and word[i + 1] == second:
                                 new_word.append(first + second)
                                 i += 2
                          else:
                                 new_word.append(word[i])
                                 i += 1
                   _A				:	int       =			tuple(__lowerCamelCase)
                   _A				:	Optional[Any]       =			new_word
                   if len(__lowerCamelCase) == 1:
                          break
                   else:
                          _A				:	List[str]       =			get_pairs(__lowerCamelCase)
            _A				:	Any       =			" ".join(__lowerCamelCase)
            _A				:	Dict       =			word
            return word


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase=None)      ->    Optional[int]:
            if self.add_bos_token:
                   _A				:	Optional[int]       =			[self.bos_token_id]
            else:
                   _A				:	Optional[int]       =			[]

            _A				:	Optional[int]       =			bos_token_ids + token_ids_a

            if token_ids_a is None:
                   return output

            return output + bos_token_ids + token_ids_a


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    List[str]:
            _A				:	Any       =			[]
            for token in re.findall(self.pat			,						__lowerCamelCase):
                   _A				:	int       =			"".join(
                       self.byte_encoder[b] for b in token.encode("utf-8"))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                   bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase).split(" "))
            return bpe_tokens


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    Optional[int]:
            return self.encoder.get(__lowerCamelCase			,						self.encoder.get(self.unk_token))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    Union[str, Any]:
            return self.decoder.get(__lowerCamelCase)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    Union[str, Any]:
            _A				:	Tuple       =			"".join(__lowerCamelCase)
            _A				:	str       =			bytearray([self.byte_decoder[c] for c in text]).decode("utf-8"			,						errors=self.errors)
            return text


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    Tuple[str]:
            if not os.path.isdir(__lowerCamelCase):
                   logger.error(F"Vocabulary path ({save_directory}) should be a directory")
                   return
            _A				:	List[str]       =			os.path.join(
                __lowerCamelCase			,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
            _A				:	List[Any]       =			os.path.join(
                __lowerCamelCase			,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

            with open(__lowerCamelCase			,						"w"			,						encoding="utf-8") as f:
                   f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=__lowerCamelCase			,						ensure_ascii=__lowerCamelCase) + "\n")

            _A				:	Optional[Any]       =			0
            with open(__lowerCamelCase			,						"w"			,						encoding="utf-8") as writer:
                   writer.write("#version: 0.2\n")
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda __lowerCamelCase: kv[1]):
                          if index != token_index:
                                 logger.warning(
                                     F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                     " Please check that the tokenizer is not corrupted!")
                                 _A				:	Dict       =			token_index
                          writer.write(" ".join(__lowerCamelCase) + "\n")
                          index += 1

            return vocab_file, merge_file


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase=False			,						**__lowerCamelCase)      ->    Optional[int]:
            _A				:	int       =			kwargs.pop("add_prefix_space"			,						self.add_prefix_space)
            if is_split_into_words or add_prefix_space:
                   _A				:	int       =			" " + text
            return (text, kwargs)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = False			,						__lowerCamelCase = None			,						__lowerCamelCase = None			,						**__lowerCamelCase			,						)      ->    str:
            _A				:	List[Any]       =			super()._decode(
                token_ids=__lowerCamelCase			,						skip_special_tokens=__lowerCamelCase			,						clean_up_tokenization_spaces=__lowerCamelCase			,						**__lowerCamelCase			,						)

            if truncate_before_pattern is not None and len(__lowerCamelCase) > 0:
                   _A				:	Any       =			self.truncate(__lowerCamelCase			,						__lowerCamelCase)

            return decoded_text


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase)      ->    Optional[int]:
            def find_re(__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase):
                   _A				:	str       =			pattern.search(__lowerCamelCase			,						__lowerCamelCase)
                   return m.start() if m else -1

            _A				:	int       =			[re.compile(__lowerCamelCase			,						re.MULTILINE) for pattern in truncate_before_pattern]

            _A				:	Optional[int]       =			list(re.finditer("^print"			,						__lowerCamelCase			,						re.MULTILINE))

            if len(__lowerCamelCase) > 1:
                   _A				:	List[Any]       =			completion[: prints[1].start()]

            _A				:	Dict       =			list(re.finditer("^def"			,						__lowerCamelCase			,						re.MULTILINE))

            if len(__lowerCamelCase) > 1:
                   _A				:	Optional[Any]       =			completion[: defs[1].start()]

            _A				:	Union[str, Any]       =			0

            _A				:	Any       =			[
                pos for pos in [find_re(__lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase) for terminal in terminals] if pos != -1
            ]

            if len(__lowerCamelCase) > 0:
                   return completion[: min(__lowerCamelCase)]
            else:
                   return completion




from __future__ import annotations

from typing import Any







class 					lowerCAmelCase__	:




     '''simple docstring'''


     def __init__(						self			,						__lowerCamelCase			,						__lowerCamelCase)      ->    None:  # noqa: A002
            _A				:	List[str]       =			item
            _A				:	List[Any]       =			next







class 					lowerCAmelCase__	:




     '''simple docstring'''


     def __init__(						self)      ->    None:
            _A				:	Node | None       =			None
            _A				:	List[Any]       =			0


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    None:
            _A				:	Union[str, Any]       =			Node(__lowerCamelCase			,						self.head)
            self.size += 1


     def 			_lowerCamelCase			(						self)      ->    Any:
            # Switched 'self.is_empty()' to 'self.head is None'
            # because mypy was considering the possibility that 'self.head'
            # can be None in below else part and giving error
            if self.head is None:
                   return None
            else:
                   _A				:	str       =			self.head.item
                   _A				:	List[Any]       =			self.head.next
                   self.size -= 1
                   return item


     def 			_lowerCamelCase			(						self)      ->    bool:
            return self.head is None


     def __str__(						self)      ->    str:
            if self.is_empty():
                   return ""
            else:
                   _A				:	List[str]       =			self.head
                   _A				:	Optional[int]       =			""
                   _A				:	list[str]       =			[]
                   while iterate:
                          item_list.append(str(iterate.item))
                          _A				:	Optional[Any]       =			iterate.next

                   _A				:	List[str]       =			" --> ".join(__lowerCamelCase)

                   return item_str


     def __len__(						self)      ->    int:
            return self.size
