




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_lowerCamelCase					:     Optional[int]				       =    logging.get_logger(__name__)

_lowerCamelCase					:     Optional[Any]				       =    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase					:     Dict				       =    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_lowerCamelCase					:     List[Any]				       =    {
    'Salesforce/codegen-350M-mono': 2048,
}

@lru_cache()
def 						lowercase_ (						):




	"""simple docstring"""






	A_					:			Optional[int]						=						(
	    list(range(ord('''!'''		)					, ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)					, ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)					, ord('''ÿ'''		) + 1		)		)
	)
	A_					:			int						=						bs[:]
	A_					:			int						=						0
	for b in range(2**8		):
		if b not in bs:
			bs.append(_UpperCAmelCase		)
			cs.append(2**8 + n		)
			n += 1
	A_					:			Tuple						=						[chr(_UpperCAmelCase		) for n in cs]
	return dict(zip(_UpperCAmelCase					, _UpperCAmelCase		)		)

def 						lowercase_ (						_UpperCAmelCase		):




	"""simple docstring"""






	A_					:			Optional[Any]						=						set()
	A_					:			Optional[Any]						=						word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)		)
		A_					:			Optional[Any]						=						char
	return pairs

class     lowercase (      __UpperCAmelCase):
				__lowerCAmelCase      :      Tuple					    =				VOCAB_FILES_NAMES
				__lowerCAmelCase      :      Union[str, Any]					    =				PRETRAINED_VOCAB_FILES_MAP
				__lowerCAmelCase      :      Optional[int]					    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCAmelCase      :      Any					    =				["""input_ids""", """attention_mask"""]







				def __init__(							self				:   List[str] ,       _lowerCamelCase				:   Tuple ,       _lowerCamelCase				:   int ,       _lowerCamelCase				:   Optional[Any]="replace" ,       _lowerCamelCase				:   str="<|endoftext|>" ,       _lowerCamelCase				:   Tuple="<|endoftext|>" ,       _lowerCamelCase				:   int="<|endoftext|>" ,       _lowerCamelCase				:   List[str]=None ,       _lowerCamelCase				:   Dict=False ,       _lowerCamelCase				:   List[Any]=False ,       **_lowerCamelCase				:   str ,       ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						AddedToken(_lowerCamelCase ,       lstrip=_lowerCamelCase ,       rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase ,       _lowerCamelCase    ) else bos_token
					A_					:			int						=						AddedToken(_lowerCamelCase ,       lstrip=_lowerCamelCase ,       rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase ,       _lowerCamelCase    ) else eos_token
					A_					:			Any						=						AddedToken(_lowerCamelCase ,       lstrip=_lowerCamelCase ,       rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase ,       _lowerCamelCase    ) else unk_token
					A_					:			Optional[Any]						=						AddedToken(_lowerCamelCase ,       lstrip=_lowerCamelCase ,       rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase ,       _lowerCamelCase    ) else pad_token
					super().__init__(
					    errors=_lowerCamelCase ,       unk_token=_lowerCamelCase ,       bos_token=_lowerCamelCase ,       eos_token=_lowerCamelCase ,       pad_token=_lowerCamelCase ,       add_prefix_space=_lowerCamelCase ,       add_bos_token=_lowerCamelCase ,       **_lowerCamelCase ,       )
					A_					:			int						=						add_bos_token

					with open(_lowerCamelCase ,       encoding='''utf-8'''    ) as vocab_handle:
						A_					:			List[Any]						=						json.load(_lowerCamelCase    )
					A_					:			Union[str, Any]						=						{v: k for k, v in self.encoder.items()}
					A_					:			Optional[int]						=						errors  # how to handle errors in decoding
					A_					:			Tuple						=						bytes_to_unicode()
					A_					:			Union[str, Any]						=						{v: k for k, v in self.byte_encoder.items()}
					with open(_lowerCamelCase ,       encoding='''utf-8'''    ) as merges_handle:
						A_					:			Any						=						merges_handle.read().split('''\n'''    )[1:-1]
					A_					:			Tuple						=						[tuple(merge.split()    ) for merge in bpe_merges]
					A_					:			Optional[Any]						=						dict(zip(_lowerCamelCase ,       range(len(_lowerCamelCase    )    )    )    )
					A_					:			Any						=						{}
					A_					:			List[str]						=						add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					A_					:			List[str]						=						re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )







				@property
				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					return len(self.encoder    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					return dict(self.encoder ,       **self.added_tokens_encoder    )







				def    a_					(							self				:   List[Any] ,       _lowerCamelCase				:   Optional[Any]    ):



					"""simple docstring"""

					if token in self.cache:
						return self.cache[token]
					A_					:			Tuple						=						tuple(_lowerCamelCase    )
					A_					:			Optional[Any]						=						get_pairs(_lowerCamelCase    )

					if not pairs:
						return token

					while True:
						A_					:			str						=						min(_lowerCamelCase ,       key=lambda _lowerCamelCase    : self.bpe_ranks.get(_lowerCamelCase ,       float('''inf'''    )    )    )
						if bigram not in self.bpe_ranks:
							break
						A_					:			Optional[int]						=						bigram
						A_					:			Dict						=						[]
						A_					:			str						=						0
						while i < len(_lowerCamelCase    ):
							try:
								A_					:			Tuple						=						word.index(_lowerCamelCase ,       _lowerCamelCase    )
							except ValueError:
								new_word.extend(word[i:]    )
								break
							else:
								new_word.extend(word[i:j]    )
								A_					:			List[str]						=						j

							if word[i] == first and i < len(_lowerCamelCase    ) - 1 and word[i + 1] == second:
								new_word.append(first + second    )
								i += 2
							else:
								new_word.append(word[i]    )
								i += 1
						A_					:			Optional[int]						=						tuple(_lowerCamelCase    )
						A_					:			Union[str, Any]						=						new_word
						if len(_lowerCamelCase    ) == 1:
							break
						else:
							A_					:			List[Any]						=						get_pairs(_lowerCamelCase    )
					A_					:			Tuple						=						''' '''.join(_lowerCamelCase    )
					A_					:			Optional[Any]						=						word
					return word







				def    a_					(							self				:   int ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   List[Any]=None    ):



					"""simple docstring"""

					if self.add_bos_token:
						A_					:			Dict						=						[self.bos_token_id]
					else:
						A_					:			Union[str, Any]						=						[]

					A_					:			Dict						=						bos_token_ids + token_ids_a

					if token_ids_a is None:
						return output

					return output + bos_token_ids + token_ids_a







				def    a_					(							self				:   Dict ,       _lowerCamelCase				:   Tuple    ):



					"""simple docstring"""

					A_					:			List[Any]						=						[]
					for token in re.findall(self.pat ,       _lowerCamelCase    ):
						A_					:			Any						=						''''''.join(
						    self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
						bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase    ).split(''' '''    )    )
					return bpe_tokens







				def    a_					(							self				:   List[str] ,       _lowerCamelCase				:   Optional[Any]    ):



					"""simple docstring"""

					return self.encoder.get(_lowerCamelCase ,       self.encoder.get(self.unk_token    )    )







				def    a_					(							self				:   Dict ,       _lowerCamelCase				:   List[Any]    ):



					"""simple docstring"""

					return self.decoder.get(_lowerCamelCase    )







				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   List[str]    ):



					"""simple docstring"""

					A_					:			Tuple						=						''''''.join(_lowerCamelCase    )
					A_					:			Optional[int]						=						bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8''' ,       errors=self.errors    )
					return text







				def    a_					(							self				:   List[Any] ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   Optional[str] = None    ):



					"""simple docstring"""

					if not os.path.isdir(_lowerCamelCase    ):
						logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
						return
					A_					:			Optional[Any]						=						os.path.join(
					    _lowerCamelCase ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
					A_					:			Any						=						os.path.join(
					    _lowerCamelCase ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

					with open(_lowerCamelCase ,       '''w''' ,       encoding='''utf-8'''    ) as f:
						f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=_lowerCamelCase ,       ensure_ascii=_lowerCamelCase    ) + '''\n'''    )

					A_					:			str						=						0
					with open(_lowerCamelCase ,       '''w''' ,       encoding='''utf-8'''    ) as writer:
						writer.write('''#version: 0.2\n'''    )
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda _lowerCamelCase    : kv[1]    ):
							if index != token_index:
								logger.warning(
								    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
								    ''' Please check that the tokenizer is not corrupted!'''    )
								A_					:			Any						=						token_index
							writer.write(''' '''.join(_lowerCamelCase    ) + '''\n'''    )
							index += 1

					return vocab_file, merge_file







				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   Optional[Any]=False ,       **_lowerCamelCase				:   Any    ):



					"""simple docstring"""

					A_					:			Dict						=						kwargs.pop('''add_prefix_space''' ,       self.add_prefix_space    )
					if is_split_into_words or add_prefix_space:
						A_					:			Any						=						''' ''' + text
					return (text, kwargs)







				def    a_					(							self				:   Tuple ,       _lowerCamelCase				:   Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"] ,       _lowerCamelCase				:   bool = False ,       _lowerCamelCase				:   bool = None ,       _lowerCamelCase				:   Optional[List[str]] = None ,       **_lowerCamelCase				:   Optional[Any] ,       ):



					"""simple docstring"""

					A_					:			Any						=						super()._decode(
					    token_ids=_lowerCamelCase ,       skip_special_tokens=_lowerCamelCase ,       clean_up_tokenization_spaces=_lowerCamelCase ,       **_lowerCamelCase ,       )

					if truncate_before_pattern is not None and len(_lowerCamelCase    ) > 0:
						A_					:			Optional[int]						=						self.truncate(_lowerCamelCase ,       _lowerCamelCase    )

					return decoded_text







				def    a_					(							self				:   str ,       _lowerCamelCase				:   Dict ,       _lowerCamelCase				:   List[Any]    ):



					"""simple docstring"""

					def find_re(_lowerCamelCase				:   str ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   str    ):
						A_					:			List[str]						=						pattern.search(_lowerCamelCase ,       _lowerCamelCase    )
						return m.start() if m else -1

					A_					:			int						=						[re.compile(_lowerCamelCase ,       re.MULTILINE    ) for pattern in truncate_before_pattern]

					A_					:			Dict						=						list(re.finditer('''^print''' ,       _lowerCamelCase ,       re.MULTILINE    )    )

					if len(_lowerCamelCase    ) > 1:
						A_					:			List[Any]						=						completion[: prints[1].start()]

					A_					:			Dict						=						list(re.finditer('''^def''' ,       _lowerCamelCase ,       re.MULTILINE    )    )

					if len(_lowerCamelCase    ) > 1:
						A_					:			Tuple						=						completion[: defs[1].start()]

					A_					:			Union[str, Any]						=						0

					A_					:			int						=						[
					    pos for pos in [find_re(_lowerCamelCase ,       _lowerCamelCase ,       _lowerCamelCase    ) for terminal in terminals] if pos != -1
					]

					if len(_lowerCamelCase    ) > 0:
						return completion[: min(_lowerCamelCase    )]
					else:
						return completion











"""simple docstring"""

import enum
import inspect
import warnings
from functools import wraps
from typing import Callable, Optional

from .logging import get_logger


_lowerCamelCase					:     Optional[int]				       =    set()
_lowerCamelCase					:     Union[str, Any]				       =    get_logger(__name__)

def 						lowercase_ (						_UpperCAmelCase = None		):




	"""simple docstring"""






	def decorator(_UpperCAmelCase		):
		global _emitted_deprecation_warnings

		if inspect.isclass(_UpperCAmelCase		):
			A_					:			Union[str, Any]						=						deprecated_class_or_function.__init__
			A_					:			int						=						deprecated_class_or_function.__name__
		else:
			A_					:			Union[str, Any]						=						deprecated_class_or_function
			A_					:			List[Any]						=						deprecated_function.__name__
			# Support deprecating __init__ class method: class name instead
			A_					:			List[Any]						=						name if name != '''__init__''' else deprecated_function.__qualname__.split('''.'''		)[-2]

		A_					:			str						=						(
		    f"""{name} is deprecated and will be removed in the next major version of datasets.""" + f""" {help_message}"""
		    if help_message
		    else ''''''
		)

		@wraps(_UpperCAmelCase		)
		def wrapper(*_UpperCAmelCase					, **_UpperCAmelCase		):
			A_					:			Any						=						hash(_UpperCAmelCase		)
			if func_hash not in _emitted_deprecation_warnings:
				warnings.warn(_UpperCAmelCase					, category=_UpperCAmelCase					, stacklevel=2		)
				_emitted_deprecation_warnings.add(_UpperCAmelCase		)
			return deprecated_function(*_UpperCAmelCase					, **_UpperCAmelCase		)

		A_					:			Any						=						'''deprecated'''

		if inspect.isclass(_UpperCAmelCase		):
			A_					:			List[Any]						=						wrapper
			return deprecated_class_or_function
		else:
			return wrapper

	return decorator

class     lowercase (      enum.EnumMeta):







				def __getattribute__(							cls				:   Any ,       _lowerCamelCase				:   List[Any]    ):



					"""simple docstring"""

					A_					:			str						=						super().__getattribute__(_lowerCamelCase    )
					if isinstance(_lowerCamelCase ,       enum.Enum    ) and obj._on_access:
						obj._on_access()
					return obj







				def __getitem__(							cls				:   Any ,       _lowerCamelCase				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						super().__getitem__(_lowerCamelCase    )
					if member._on_access:
						member._on_access()
					return member







				def __call__(							cls				:   str ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   List[Any]=None ,       *, _lowerCamelCase				:   int=None ,       _lowerCamelCase				:   str=None ,       _lowerCamelCase				:   Dict=None ,       _lowerCamelCase				:   Tuple=1    ):



					"""simple docstring"""

					A_					:			Dict						=						super().__call__(_lowerCamelCase ,       _lowerCamelCase ,       module=_lowerCamelCase ,       qualname=_lowerCamelCase ,       type=_lowerCamelCase ,       start=_lowerCamelCase    )
					if isinstance(_lowerCamelCase ,       enum.Enum    ) and obj._on_access:
						obj._on_access()
					return obj

class     lowercase (      enum.Enum       ,		metaclass=__UpperCAmelCase):







				def __new__(							cls				:   Optional[Any] ,       _lowerCamelCase				:   int    ):



					"""simple docstring"""

					A_					:			Tuple						=						object.__new__(cls    )
					A_					:			Union[str, Any]						=						value
					A_					:			Union[str, Any]						=						member.deprecate
					return member







				@property
				def    a_					(							self				:   Tuple    ):



					"""simple docstring"""

					return ""







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						F""" {self.help_message}""" if self.help_message else ''''''
					warnings.warn(
					    F"""'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets."""
					    + help_message ,       _lowerCamelCase ,       stacklevel=3 ,       )





