

from typing import List, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCamelCase			      =		logging.get_logger(__name__)

_UpperCamelCase			      =		{
    "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/config.json",
    # See all LED models at https://huggingface.co/models?filter=led
}







class   __lowercase				(_UpperCAmelCase			):
 _UpperCamelCase        =  """led"""
 _UpperCamelCase        =  {
     """num_attention_heads""": """encoder_attention_heads""",
     """hidden_size""": """d_model""",
     """attention_probs_dropout_prob""": """attention_dropout""",
     """initializer_range""": """init_std""",
 }
 def __init__(							self				,							A_=5_0265				,							A_=1_6384				,							A_=1024				,							A_=12				,							A_=4096				,							A_=16				,							A_=12				,							A_=4096				,							A_=16				,							A_=0.0				,							A_=0.0				,							A_=True				,							A_=True				,							A_="gelu"				,							A_=1024				,							A_=0.1				,							A_=0.0				,							A_=0.0				,							A_=0.02				,							A_=2				,							A_=0.0				,							A_=1				,							A_=0				,							A_=2				,							A_ = 512				,							**A_				,							) ->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Any = vocab_size
       __lowerCAmelCase :						List[str] = max_encoder_position_embeddings
       __lowerCAmelCase :						str = max_decoder_position_embeddings
       __lowerCAmelCase :						Union[str, Any] = d_model
       __lowerCAmelCase :						str = encoder_ffn_dim
       __lowerCAmelCase :						List[Any] = encoder_layers
       __lowerCAmelCase :						int = encoder_attention_heads
       __lowerCAmelCase :						List[str] = decoder_ffn_dim
       __lowerCAmelCase :						Any = decoder_layers
       __lowerCAmelCase :						int = decoder_attention_heads
       __lowerCAmelCase :						int = dropout
       __lowerCAmelCase :						Dict = attention_dropout
       __lowerCAmelCase :						Tuple = activation_dropout
       __lowerCAmelCase :						Any = activation_function
       __lowerCAmelCase :						Dict = init_std
       __lowerCAmelCase :						int = encoder_layerdrop
       __lowerCAmelCase :						int = decoder_layerdrop
       __lowerCAmelCase :						Tuple = classifier_dropout
       __lowerCAmelCase :						Any = use_cache
       __lowerCAmelCase :						Dict = encoder_layers
       __lowerCAmelCase :						int = attention_window

       super().__init__(
           pad_token_id=A_				,							bos_token_id=A_				,							eos_token_id=A_				,							is_encoder_decoder=A_				,							decoder_start_token_id=A_				,							**A_				,							)





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_UpperCamelCase			      =		logging.get_logger(__name__)

_UpperCamelCase			      =		{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_UpperCamelCase			      =		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

_UpperCamelCase			      =		{
    "Salesforce/codegen-350M-mono": 2048,
}







@lru_cache()
def 		_lowercase			(					):
      __lowerCAmelCase :						Union[str, Any] = (
          list(range(ord('''!'''						)       ,     ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)       ,     ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)       ,     ord('''ÿ'''						) + 1						)						)
      )
      __lowerCAmelCase :						Optional[int] = bs[:]
      __lowerCAmelCase :						Union[str, Any] = 0
      for b in range(2**8						):
            if b not in bs:
                  bs.append(lowercase__						)
                  cs.append(2**8 + n						)
                  n += 1
      __lowerCAmelCase :						Optional[int] = [chr(lowercase__						) for n in cs]
      return dict(zip(lowercase__       ,     lowercase__						)						)







def 		_lowercase			(					lowercase__						):
      __lowerCAmelCase :						List[Any] = set()
      __lowerCAmelCase :						int = word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)						)
            __lowerCAmelCase :						int = char
      return pairs







class   __lowercase				(_UpperCAmelCase			):
 _UpperCamelCase        =  VOCAB_FILES_NAMES
 _UpperCamelCase        =  PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase        =  ["""input_ids""", """attention_mask"""]
 def __init__(							self				,							A_				,							A_				,							A_="replace"				,							A_="<|endoftext|>"				,							A_="<|endoftext|>"				,							A_="<|endoftext|>"				,							A_=None				,							A_=False				,							A_=False				,							**A_				,							) ->Union[str, Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Union[str, Any] = AddedToken(A_				,							lstrip=A_				,							rstrip=A_ ) if isinstance(A_				,							A_ ) else bos_token
       __lowerCAmelCase :						Tuple = AddedToken(A_				,							lstrip=A_				,							rstrip=A_ ) if isinstance(A_				,							A_ ) else eos_token
       __lowerCAmelCase :						Any = AddedToken(A_				,							lstrip=A_				,							rstrip=A_ ) if isinstance(A_				,							A_ ) else unk_token
       __lowerCAmelCase :						List[Any] = AddedToken(A_				,							lstrip=A_				,							rstrip=A_ ) if isinstance(A_				,							A_ ) else pad_token
       super().__init__(
           errors=A_				,							unk_token=A_				,							bos_token=A_				,							eos_token=A_				,							pad_token=A_				,							add_prefix_space=A_				,							add_bos_token=A_				,							**A_				,							)
       __lowerCAmelCase :						List[str] = add_bos_token

       with open(A_				,							encoding='''utf-8''' ) as vocab_handle:
             __lowerCAmelCase :						Optional[int] = json.load(A_ )
       __lowerCAmelCase :						str = {v: k for k, v in self.encoder.items()}
       __lowerCAmelCase :						Optional[Any] = errors  # how to handle errors in decoding
       __lowerCAmelCase :						Optional[int] = bytes_to_unicode()
       __lowerCAmelCase :						Optional[Any] = {v: k for k, v in self.byte_encoder.items()}
       with open(A_				,							encoding='''utf-8''' ) as merges_handle:
             __lowerCAmelCase :						Optional[int] = merges_handle.read().split('''\n''' )[1:-1]
       __lowerCAmelCase :						List[Any] = [tuple(merge.split() ) for merge in bpe_merges]
       __lowerCAmelCase :						List[Any] = dict(zip(A_				,							range(len(A_ ) ) ) )
       __lowerCAmelCase :						Any = {}
       __lowerCAmelCase :						Optional[int] = add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       __lowerCAmelCase :						Optional[Any] = re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''' )
 @property
 def      UpperCamelCase__  (							self ) ->Dict:



       '''simple docstring'''





       return len(self.encoder )
 def      UpperCamelCase__  (							self ) ->str:



       '''simple docstring'''





       return dict(self.encoder				,							**self.added_tokens_encoder )
 def      UpperCamelCase__  (							self				,							A_ ) ->Tuple:



       '''simple docstring'''





       if token in self.cache:
             return self.cache[token]
       __lowerCAmelCase :						Tuple = tuple(A_ )
       __lowerCAmelCase :						str = get_pairs(A_ )

       if not pairs:
             return token

       while True:
             __lowerCAmelCase :						int = min(A_				,							key=lambda A_ : self.bpe_ranks.get(A_				,							float('''inf''' ) ) )
             if bigram not in self.bpe_ranks:
                   break
             __lowerCAmelCase,							__lowerCAmelCase :						Dict = bigram
             __lowerCAmelCase :						Union[str, Any] = []
             __lowerCAmelCase :						Union[str, Any] = 0
             while i < len(A_ ):
                   try:
                         __lowerCAmelCase :						List[str] = word.index(A_				,							A_ )
                   except ValueError:
                         new_word.extend(word[i:] )
                         break
                   else:
                         new_word.extend(word[i:j] )
                         __lowerCAmelCase :						Optional[int] = j

                   if word[i] == first and i < len(A_ ) - 1 and word[i + 1] == second:
                         new_word.append(first + second )
                         i += 2
                   else:
                         new_word.append(word[i] )
                         i += 1
             __lowerCAmelCase :						Optional[int] = tuple(A_ )
             __lowerCAmelCase :						List[Any] = new_word
             if len(A_ ) == 1:
                   break
             else:
                   __lowerCAmelCase :						Optional[int] = get_pairs(A_ )
       __lowerCAmelCase :						List[str] = ''' '''.join(A_ )
       __lowerCAmelCase :						Any = word
       return word
 def      UpperCamelCase__  (							self				,							A_				,							A_=None ) ->Tuple:



       '''simple docstring'''





       if self.add_bos_token:
             __lowerCAmelCase :						Any = [self.bos_token_id]
       else:
             __lowerCAmelCase :						Union[str, Any] = []

       __lowerCAmelCase :						Union[str, Any] = bos_token_ids + token_ids_a

       if token_ids_a is None:
             return output

       return output + bos_token_ids + token_ids_a
 def      UpperCamelCase__  (							self				,							A_ ) ->List[str]:



       '''simple docstring'''





       __lowerCAmelCase :						str = []
       for token in re.findall(self.pat				,							A_ ):
             __lowerCAmelCase :						Any = ''''''.join(
                 self.byte_encoder[b] for b in token.encode('''utf-8''' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_ ).split(''' ''' ) )
       return bpe_tokens
 def      UpperCamelCase__  (							self				,							A_ ) ->Dict:



       '''simple docstring'''





       return self.encoder.get(A_				,							self.encoder.get(self.unk_token ) )
 def      UpperCamelCase__  (							self				,							A_ ) ->Optional[int]:



       '''simple docstring'''





       return self.decoder.get(A_ )
 def      UpperCamelCase__  (							self				,							A_ ) ->Optional[Any]:



       '''simple docstring'''





       __lowerCAmelCase :						List[Any] = ''''''.join(A_ )
       __lowerCAmelCase :						Optional[Any] = bytearray([self.byte_decoder[c] for c in text] ).decode('''utf-8'''				,							errors=self.errors )
       return text
 def      UpperCamelCase__  (							self				,							A_				,							A_ = None ) ->Tuple[str]:



       '''simple docstring'''





       if not os.path.isdir(A_ ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
             return
       __lowerCAmelCase :						List[str] = os.path.join(
           A_				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
       __lowerCAmelCase :						Tuple = os.path.join(
           A_				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

       with open(A_				,							'''w'''				,							encoding='''utf-8''' ) as f:
             f.write(json.dumps(self.encoder				,							indent=2				,							sort_keys=A_				,							ensure_ascii=A_ ) + '''\n''' )

       __lowerCAmelCase :						Optional[int] = 0
       with open(A_				,							'''w'''				,							encoding='''utf-8''' ) as writer:
             writer.write('''#version: 0.2\n''' )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,							key=lambda A_ : kv[1] ):
                   if index != token_index:
                         logger.warning(
                             f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                             ''' Please check that the tokenizer is not corrupted!''' )
                         __lowerCAmelCase :						List[Any] = token_index
                   writer.write(''' '''.join(A_ ) + '''\n''' )
                   index += 1

       return vocab_file, merge_file
 def      UpperCamelCase__  (							self				,							A_				,							A_=False				,							**A_ ) ->Union[str, Any]:



       '''simple docstring'''





       __lowerCAmelCase :						Tuple = kwargs.pop('''add_prefix_space'''				,							self.add_prefix_space )
       if is_split_into_words or add_prefix_space:
             __lowerCAmelCase :						List[str] = ''' ''' + text
       return (text, kwargs)
 def      UpperCamelCase__  (							self				,							A_				,							A_ = False				,							A_ = None				,							A_ = None				,							**A_				,							) ->str:



       '''simple docstring'''





       __lowerCAmelCase :						List[Any] = super()._decode(
           token_ids=A_				,							skip_special_tokens=A_				,							clean_up_tokenization_spaces=A_				,							**A_				,							)

       if truncate_before_pattern is not None and len(A_ ) > 0:
             __lowerCAmelCase :						str = self.truncate(A_				,							A_ )

       return decoded_text





 def      UpperCamelCase__  (							self				,							A_				,							A_ ) ->int:



       '''simple docstring'''





       def find_re(A_				,							A_				,							A_ ):
             __lowerCAmelCase :						Dict = pattern.search(A_				,							A_ )
             return m.start() if m else -1

       __lowerCAmelCase :						int = [re.compile(A_				,							re.MULTILINE ) for pattern in truncate_before_pattern]

       __lowerCAmelCase :						Tuple = list(re.finditer('''^print'''				,							A_				,							re.MULTILINE ) )

       if len(A_ ) > 1:
             __lowerCAmelCase :						int = completion[: prints[1].start()]

       __lowerCAmelCase :						Optional[Any] = list(re.finditer('''^def'''				,							A_				,							re.MULTILINE ) )

       if len(A_ ) > 1:
             __lowerCAmelCase :						List[str] = completion[: defs[1].start()]

       __lowerCAmelCase :						Optional[Any] = 0

       __lowerCAmelCase :						Optional[int] = [
           pos for pos in [find_re(A_				,							A_				,							A_ ) for terminal in terminals] if pos != -1
       ]

       if len(A_ ) > 0:
             return completion[: min(A_ )]
       else:
             return completion


