




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      :				Tuple				      ={
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__SCREAMING_SNAKE_CASE      :				Optional[int]				      ={
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__SCREAMING_SNAKE_CASE      :				int				      ={
    """Salesforce/codegen-350M-mono""": 2_048,
}

@lru_cache()
def      lowerCAmelCase_     (					)   ->     Any:


      '''simple docstring'''




      lowerCamelCase__ 					=			(
          list(range(ord('!'  )       ,			ord('~'  ) + 1  )  ) + list(range(ord('¡'  )       ,			ord('¬'  ) + 1  )  ) + list(range(ord('®'  )       ,			ord('ÿ'  ) + 1  )  )
      )
      lowerCamelCase__ 					=			bs[:]
      lowerCamelCase__ 					=			0
      for b in range(2**8  ):
            if b not in bs:
                  bs.append(lowerCamelCase  )
                  cs.append(2**8 + n  )
                  n += 1
      lowerCamelCase__ 					=			[chr(lowerCamelCase  ) for n in cs]
      return dict(zip(lowerCamelCase       ,			lowerCamelCase  )  )

def      lowerCAmelCase_     (					lowerCamelCase:    int  )   ->     Optional[int]:


      '''simple docstring'''




      lowerCamelCase__ 					=			set()
      lowerCamelCase__ 					=			word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)  )
            lowerCamelCase__ 					=			char
      return pairs






class      A     (						_lowerCAmelCase						):
     """simple docstring"""

     _snake_case:		int       =VOCAB_FILES_NAMES
     _snake_case:		Union[str, Any]       =PRETRAINED_VOCAB_FILES_MAP
     _snake_case:		Optional[Any]       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _snake_case:		Tuple       =["input_ids", "attention_mask"]




     def __init__(      self					:				str		,					A__					:				Union[str, Any]		,					A__					:				Dict		,					A__					:				Union[str, Any]="replace"		,					A__					:				Optional[int]="<|endoftext|>"		,					A__					:				Tuple="<|endoftext|>"		,					A__					:				Optional[int]="<|endoftext|>"		,					A__					:				Tuple=None		,					A__					:				List[Any]=False		,					A__					:				Dict=False		,					**A__					:				str		,					)			->    int:
           lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else bos_token
           lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else eos_token
           lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else unk_token
           lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else pad_token
           super().__init__(
               errors=A__		,					unk_token=A__		,					bos_token=A__		,					eos_token=A__		,					pad_token=A__		,					add_prefix_space=A__		,					add_bos_token=A__		,					**A__		,					)
           lowerCamelCase__ 					=			add_bos_token

           with open(A__		,					encoding='utf-8'     ) as vocab_handle:
                 lowerCamelCase__ 					=			json.load(A__     )
           lowerCamelCase__ 					=			{v: k for k, v in self.encoder.items()}
           lowerCamelCase__ 					=			errors  # how to handle errors in decoding
           lowerCamelCase__ 					=			bytes_to_unicode()
           lowerCamelCase__ 					=			{v: k for k, v in self.byte_encoder.items()}
           with open(A__		,					encoding='utf-8'     ) as merges_handle:
                 lowerCamelCase__ 					=			merges_handle.read().split('\n'     )[1:-1]
           lowerCamelCase__ 					=			[tuple(merge.split()     ) for merge in bpe_merges]
           lowerCamelCase__ 					=			dict(zip(A__		,					range(len(A__     )     )     )     )
           lowerCamelCase__ 					=			{}
           lowerCamelCase__ 					=			add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           lowerCamelCase__ 					=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )




     @property
     def 	__snake_case				(      self					:				str     )			->    Any:
           return len(self.encoder     )




     def 	__snake_case				(      self					:				List[str]     )			->    Tuple:
           return dict(self.encoder		,					**self.added_tokens_encoder     )




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				int     )			->    int:
           if token in self.cache:
                 return self.cache[token]
           lowerCamelCase__ 					=			tuple(A__     )
           lowerCamelCase__ 					=			get_pairs(A__     )

           if not pairs:
                 return token

           while True:
                 lowerCamelCase__ 					=			min(A__		,					key=lambda A__     : self.bpe_ranks.get(A__		,					float('inf'     )     )     )
                 if bigram not in self.bpe_ranks:
                       break
                 lowerCamelCase__					,						lowerCamelCase__ 					=			bigram
                 lowerCamelCase__ 					=			[]
                 lowerCamelCase__ 					=			0
                 while i < len(A__     ):
                       try:
                             lowerCamelCase__ 					=			word.index(A__		,					A__     )
                       except ValueError:
                             new_word.extend(word[i:]     )
                             break
                       else:
                             new_word.extend(word[i:j]     )
                             lowerCamelCase__ 					=			j

                       if word[i] == first and i < len(A__     ) - 1 and word[i + 1] == second:
                             new_word.append(first + second     )
                             i += 2
                       else:
                             new_word.append(word[i]     )
                             i += 1
                 lowerCamelCase__ 					=			tuple(A__     )
                 lowerCamelCase__ 					=			new_word
                 if len(A__     ) == 1:
                       break
                 else:
                       lowerCamelCase__ 					=			get_pairs(A__     )
           lowerCamelCase__ 					=			' '.join(A__     )
           lowerCamelCase__ 					=			word
           return word




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				Tuple		,					A__					:				Dict=None     )			->    Optional[int]:
           if self.add_bos_token:
                 lowerCamelCase__ 					=			[self.bos_token_id]
           else:
                 lowerCamelCase__ 					=			[]

           lowerCamelCase__ 					=			bos_token_ids + token_ids_a

           if token_ids_a is None:
                 return output

           return output + bos_token_ids + token_ids_a




     def 	__snake_case				(      self					:				List[str]		,					A__					:				Optional[Any]     )			->    Any:
           lowerCamelCase__ 					=			[]
           for token in re.findall(self.pat		,					A__     ):
                 lowerCamelCase__ 					=			''.join(
                     self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A__     ).split(' '     )     )
           return bpe_tokens




     def 	__snake_case				(      self					:				int		,					A__					:				Any     )			->    Optional[int]:
           return self.encoder.get(A__		,					self.encoder.get(self.unk_token     )     )




     def 	__snake_case				(      self					:				Optional[int]		,					A__					:				Any     )			->    Dict:
           return self.decoder.get(A__     )




     def 	__snake_case				(      self					:				Any		,					A__					:				List[Any]     )			->    Optional[Any]:
           lowerCamelCase__ 					=			''.join(A__     )
           lowerCamelCase__ 					=			bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'		,					errors=self.errors     )
           return text




     def 	__snake_case				(      self					:				List[str]		,					A__					:				str		,					A__					:				Optional[str] = None     )			->    Tuple[str]:
           if not os.path.isdir(A__     ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
                 return
           lowerCamelCase__ 					=			os.path.join(
               A__		,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
           lowerCamelCase__ 					=			os.path.join(
               A__		,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

           with open(A__		,					'w'		,					encoding='utf-8'     ) as f:
                 f.write(json.dumps(self.encoder		,					indent=2		,					sort_keys=A__		,					ensure_ascii=A__     ) + '\n'     )

           lowerCamelCase__ 					=			0
           with open(A__		,					'w'		,					encoding='utf-8'     ) as writer:
                 writer.write('#version: 0.2\n'     )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,					key=lambda A__     : kv[1]     ):
                       if index != token_index:
                             logger.warning(
                                 F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                 ' Please check that the tokenizer is not corrupted!'     )
                             lowerCamelCase__ 					=			token_index
                       writer.write(' '.join(A__     ) + '\n'     )
                       index += 1

           return vocab_file, merge_file




     def 	__snake_case				(      self					:				Dict		,					A__					:				Union[str, Any]		,					A__					:				str=False		,					**A__					:				Optional[Any]     )			->    Optional[int]:
           lowerCamelCase__ 					=			kwargs.pop('add_prefix_space'		,					self.add_prefix_space     )
           if is_split_into_words or add_prefix_space:
                 lowerCamelCase__ 					=			' ' + text
           return (text, kwargs)




     def 	__snake_case				(      self					:				List[str]		,					A__					:				Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]		,					A__					:				bool = False		,					A__					:				bool = None		,					A__					:				Optional[List[str]] = None		,					**A__					:				Tuple		,					)			->    str:
           lowerCamelCase__ 					=			super()._decode(
               token_ids=A__		,					skip_special_tokens=A__		,					clean_up_tokenization_spaces=A__		,					**A__		,					)

           if truncate_before_pattern is not None and len(A__     ) > 0:
                 lowerCamelCase__ 					=			self.truncate(A__		,					A__     )

           return decoded_text




     def 	__snake_case				(      self					:				List[str]		,					A__					:				List[Any]		,					A__					:				int     )			->    List[Any]:
           def find_re(A__					:				str		,					A__					:				Dict		,					A__					:				str     ):
                 lowerCamelCase__ 					=			pattern.search(A__		,					A__     )
                 return m.start() if m else -1

           lowerCamelCase__ 					=			[re.compile(A__		,					re.MULTILINE     ) for pattern in truncate_before_pattern]

           lowerCamelCase__ 					=			list(re.finditer('^print'		,					A__		,					re.MULTILINE     )     )

           if len(A__     ) > 1:
                 lowerCamelCase__ 					=			completion[: prints[1].start()]

           lowerCamelCase__ 					=			list(re.finditer('^def'		,					A__		,					re.MULTILINE     )     )

           if len(A__     ) > 1:
                 lowerCamelCase__ 					=			completion[: defs[1].start()]

           lowerCamelCase__ 					=			0

           lowerCamelCase__ 					=			[
               pos for pos in [find_re(A__		,					A__		,					A__     ) for terminal in terminals] if pos != -1
           ]

           if len(A__     ) > 0:
                 return completion[: min(A__     )]
           else:
                 return completion





from __future__ import annotations

def      lowerCAmelCase_     (					lowerCamelCase:    list[int | float]  )   ->     int | float:


      '''simple docstring'''




      if len(lowerCamelCase  ) == 0:
            raise ValueError('find_max() arg is an empty sequence'  )
      lowerCamelCase__ 					=			nums[0]
      for x in nums:
            if x > max_num:
                  lowerCamelCase__ 					=			x
      return max_num


if __name__ == "__main__":
      import doctest

      doctest.testmod(verbose=True)