




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      :				Tuple				      ={
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__SCREAMING_SNAKE_CASE      :				Optional[int]				      ={
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__SCREAMING_SNAKE_CASE      :				int				      ={
    """Salesforce/codegen-350M-mono""": 2_048,
}

@lru_cache()
def      lowerCAmelCase_     (					)   ->     Any:


 '''simple docstring'''




 lowerCamelCase__ 					=			(
     list(range(ord('!'  )       ,			ord('~'  ) + 1  )  ) + list(range(ord('¡'  )       ,			ord('¬'  ) + 1  )  ) + list(range(ord('®'  )       ,			ord('ÿ'  ) + 1  )  )
 )
 lowerCamelCase__ 					=			bs[:]
 lowerCamelCase__ 					=			0
 for b in range(2**8  ):
  if b not in bs:
   bs.append(lowerCamelCase  )
   cs.append(2**8 + n  )
   n += 1
 lowerCamelCase__ 					=			[chr(lowerCamelCase  ) for n in cs]
 return dict(zip(lowerCamelCase       ,			lowerCamelCase  )  )

def      lowerCAmelCase_     (					lowerCamelCase:    int  )   ->     Optional[int]:


 '''simple docstring'''




 lowerCamelCase__ 					=			set()
 lowerCamelCase__ 					=			word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)  )
  lowerCamelCase__ 					=			char
 return pairs






class      A     (						_lowerCAmelCase						):
     """simple docstring"""

     _snake_case:		int       =VOCAB_FILES_NAMES
     _snake_case:		Union[str, Any]       =PRETRAINED_VOCAB_FILES_MAP
     _snake_case:		Optional[Any]       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _snake_case:		Tuple       =["input_ids", "attention_mask"]




     def __init__(      self					:				str		,					A__					:				Union[str, Any]		,					A__					:				Dict		,					A__					:				Union[str, Any]="replace"		,					A__					:				Optional[int]="<|endoftext|>"		,					A__					:				Tuple="<|endoftext|>"		,					A__					:				Optional[int]="<|endoftext|>"		,					A__					:				Tuple=None		,					A__					:				List[Any]=False		,					A__					:				Dict=False		,					**A__					:				str		,					)			->    int:
      lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else bos_token
      lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else eos_token
      lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else unk_token
      lowerCamelCase__ 					=			AddedToken(A__		,					lstrip=A__		,					rstrip=A__     ) if isinstance(A__		,					A__     ) else pad_token
      super().__init__(
          errors=A__		,					unk_token=A__		,					bos_token=A__		,					eos_token=A__		,					pad_token=A__		,					add_prefix_space=A__		,					add_bos_token=A__		,					**A__		,					)
      lowerCamelCase__ 					=			add_bos_token

      with open(A__		,					encoding='utf-8'     ) as vocab_handle:
       lowerCamelCase__ 					=			json.load(A__     )
      lowerCamelCase__ 					=			{v: k for k, v in self.encoder.items()}
      lowerCamelCase__ 					=			errors  # how to handle errors in decoding
      lowerCamelCase__ 					=			bytes_to_unicode()
      lowerCamelCase__ 					=			{v: k for k, v in self.byte_encoder.items()}
      with open(A__		,					encoding='utf-8'     ) as merges_handle:
       lowerCamelCase__ 					=			merges_handle.read().split('\n'     )[1:-1]
      lowerCamelCase__ 					=			[tuple(merge.split()     ) for merge in bpe_merges]
      lowerCamelCase__ 					=			dict(zip(A__		,					range(len(A__     )     )     )     )
      lowerCamelCase__ 					=			{}
      lowerCamelCase__ 					=			add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      lowerCamelCase__ 					=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )




     @property
     def 	__snake_case				(      self					:				str     )			->    Any:
      return len(self.encoder     )




     def 	__snake_case				(      self					:				List[str]     )			->    Tuple:
      return dict(self.encoder		,					**self.added_tokens_encoder     )




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				int     )			->    int:
      if token in self.cache:
       return self.cache[token]
      lowerCamelCase__ 					=			tuple(A__     )
      lowerCamelCase__ 					=			get_pairs(A__     )

      if not pairs:
       return token

      while True:
       lowerCamelCase__ 					=			min(A__		,					key=lambda A__     : self.bpe_ranks.get(A__		,					float('inf'     )     )     )
       if bigram not in self.bpe_ranks:
        break
       lowerCamelCase__					,						lowerCamelCase__ 					=			bigram
       lowerCamelCase__ 					=			[]
       lowerCamelCase__ 					=			0
       while i < len(A__     ):
        try:
         lowerCamelCase__ 					=			word.index(A__		,					A__     )
        except ValueError:
         new_word.extend(word[i:]     )
         break
        else:
         new_word.extend(word[i:j]     )
         lowerCamelCase__ 					=			j

        if word[i] == first and i < len(A__     ) - 1 and word[i + 1] == second:
         new_word.append(first + second     )
         i += 2
        else:
         new_word.append(word[i]     )
         i += 1
       lowerCamelCase__ 					=			tuple(A__     )
       lowerCamelCase__ 					=			new_word
       if len(A__     ) == 1:
        break
       else:
        lowerCamelCase__ 					=			get_pairs(A__     )
      lowerCamelCase__ 					=			' '.join(A__     )
      lowerCamelCase__ 					=			word
      return word




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				Tuple		,					A__					:				Dict=None     )			->    Optional[int]:
      if self.add_bos_token:
       lowerCamelCase__ 					=			[self.bos_token_id]
      else:
       lowerCamelCase__ 					=			[]

      lowerCamelCase__ 					=			bos_token_ids + token_ids_a

      if token_ids_a is None:
       return output

      return output + bos_token_ids + token_ids_a




     def 	__snake_case				(      self					:				List[str]		,					A__					:				Optional[Any]     )			->    Any:
      lowerCamelCase__ 					=			[]
      for token in re.findall(self.pat		,					A__     ):
       lowerCamelCase__ 					=			''.join(
           self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A__     ).split(' '     )     )
      return bpe_tokens




     def 	__snake_case				(      self					:				int		,					A__					:				Any     )			->    Optional[int]:
      return self.encoder.get(A__		,					self.encoder.get(self.unk_token     )     )




     def 	__snake_case				(      self					:				Optional[int]		,					A__					:				Any     )			->    Dict:
      return self.decoder.get(A__     )




     def 	__snake_case				(      self					:				Any		,					A__					:				List[Any]     )			->    Optional[Any]:
      lowerCamelCase__ 					=			''.join(A__     )
      lowerCamelCase__ 					=			bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'		,					errors=self.errors     )
      return text




     def 	__snake_case				(      self					:				List[str]		,					A__					:				str		,					A__					:				Optional[str] = None     )			->    Tuple[str]:
      if not os.path.isdir(A__     ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
       return
      lowerCamelCase__ 					=			os.path.join(
          A__		,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
      lowerCamelCase__ 					=			os.path.join(
          A__		,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

      with open(A__		,					'w'		,					encoding='utf-8'     ) as f:
       f.write(json.dumps(self.encoder		,					indent=2		,					sort_keys=A__		,					ensure_ascii=A__     ) + '\n'     )

      lowerCamelCase__ 					=			0
      with open(A__		,					'w'		,					encoding='utf-8'     ) as writer:
       writer.write('#version: 0.2\n'     )
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,					key=lambda A__     : kv[1]     ):
        if index != token_index:
         logger.warning(
             F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
             ' Please check that the tokenizer is not corrupted!'     )
         lowerCamelCase__ 					=			token_index
        writer.write(' '.join(A__     ) + '\n'     )
        index += 1

      return vocab_file, merge_file




     def 	__snake_case				(      self					:				Dict		,					A__					:				Union[str, Any]		,					A__					:				str=False		,					**A__					:				Optional[Any]     )			->    Optional[int]:
      lowerCamelCase__ 					=			kwargs.pop('add_prefix_space'		,					self.add_prefix_space     )
      if is_split_into_words or add_prefix_space:
       lowerCamelCase__ 					=			' ' + text
      return (text, kwargs)




     def 	__snake_case				(      self					:				List[str]		,					A__					:				Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]		,					A__					:				bool = False		,					A__					:				bool = None		,					A__					:				Optional[List[str]] = None		,					**A__					:				Tuple		,					)			->    str:
      lowerCamelCase__ 					=			super()._decode(
          token_ids=A__		,					skip_special_tokens=A__		,					clean_up_tokenization_spaces=A__		,					**A__		,					)

      if truncate_before_pattern is not None and len(A__     ) > 0:
       lowerCamelCase__ 					=			self.truncate(A__		,					A__     )

      return decoded_text




     def 	__snake_case				(      self					:				List[str]		,					A__					:				List[Any]		,					A__					:				int     )			->    List[Any]:
      def find_re(A__					:				str		,					A__					:				Dict		,					A__					:				str     ):
       lowerCamelCase__ 					=			pattern.search(A__		,					A__     )
       return m.start() if m else -1

      lowerCamelCase__ 					=			[re.compile(A__		,					re.MULTILINE     ) for pattern in truncate_before_pattern]

      lowerCamelCase__ 					=			list(re.finditer('^print'		,					A__		,					re.MULTILINE     )     )

      if len(A__     ) > 1:
       lowerCamelCase__ 					=			completion[: prints[1].start()]

      lowerCamelCase__ 					=			list(re.finditer('^def'		,					A__		,					re.MULTILINE     )     )

      if len(A__     ) > 1:
       lowerCamelCase__ 					=			completion[: defs[1].start()]

      lowerCamelCase__ 					=			0

      lowerCamelCase__ 					=			[
          pos for pos in [find_re(A__		,					A__		,					A__     ) for terminal in terminals] if pos != -1
      ]

      if len(A__     ) > 0:
       return completion[: min(A__     )]
      else:
       return completion





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__SCREAMING_SNAKE_CASE      :				int				      ={"""configuration_nat""": ["""NAT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """NatConfig"""]}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __SCREAMING_SNAKE_CASE      :				Optional[int]				      =[
          """NAT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """NatForImageClassification""",
          """NatModel""",
          """NatPreTrainedModel""",
          """NatBackbone""",
      ]

if TYPE_CHECKING:
      from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_nat import (
                NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                NatBackbone,
                NatForImageClassification,
                NatModel,
                NatPreTrainedModel,
            )

else:
      import sys

      __SCREAMING_SNAKE_CASE      :				Any				      =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)