




'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


A			   =logging.get_logger(__name__)

A			   ={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A			   ={
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

A			   ={
    'Salesforce/codegen-350M-mono': 20_48,
}





@lru_cache()
def 						snake_case_							():
    UpperCAmelCase      	=		(
        list(range(ord('''!'''					)   ,       ord('''~'''					) + 1					)					) + list(range(ord('''¡'''					)   ,       ord('''¬'''					) + 1					)					) + list(range(ord('''®'''					)   ,       ord('''ÿ'''					) + 1					)					)
    )
    UpperCAmelCase      	=		bs[:]
    UpperCAmelCase      	=		0
    for b in range(2**8					):
        if b not in bs:
            bs.append(_a					)
            cs.append(2**8 + n					)
            n += 1
    UpperCAmelCase      	=		[chr(_a					) for n in cs]
    return dict(zip(_a   ,       _a					)					)





def 						snake_case_							(_a     :							Optional[Any]					):
    UpperCAmelCase      	=		set()
    UpperCAmelCase      	=		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)					)
        UpperCAmelCase      	=		char
    return pairs







class       _a  (     __a			):
       __a    :					Tuple           = VOCAB_FILES_NAMES
       __a    :					List[Any]           = PRETRAINED_VOCAB_FILES_MAP
       __a    :					Optional[int]           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __a    :					Dict           = ["""input_ids""", """attention_mask"""]
       def __init__(     self						:     Dict       ,    lowercase						:     Optional[Any]       ,    lowercase						:     List[str]       ,    lowercase						:     Any="replace"       ,    lowercase						:     Union[str, Any]="<|endoftext|>"       ,    lowercase						:     List[Any]="<|endoftext|>"       ,    lowercase						:     Dict="<|endoftext|>"       ,    lowercase						:     List[str]=None       ,    lowercase						:     List[Any]=False       ,    lowercase						:     List[Any]=False       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else bos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else eos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else unk_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else pad_token
           super().__init__(
               errors=lowercase       ,    unk_token=lowercase       ,    bos_token=lowercase       ,    eos_token=lowercase       ,    pad_token=lowercase       ,    add_prefix_space=lowercase       ,    add_bos_token=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		add_bos_token

           with open(lowercase       ,    encoding='''utf-8'''    ) as vocab_handle:
               UpperCAmelCase      	=		json.load(lowercase    )
           UpperCAmelCase      	=		{v: k for k, v in self.encoder.items()}
           UpperCAmelCase      	=		errors  # how to handle errors in decoding
           UpperCAmelCase      	=		bytes_to_unicode()
           UpperCAmelCase      	=		{v: k for k, v in self.byte_encoder.items()}
           with open(lowercase       ,    encoding='''utf-8'''    ) as merges_handle:
               UpperCAmelCase      	=		merges_handle.read().split('''\n'''    )[1:-1]
           UpperCAmelCase      	=		[tuple(merge.split()    ) for merge in bpe_merges]
           UpperCAmelCase      	=		dict(zip(lowercase       ,    range(len(lowercase    )    )    )    )
           UpperCAmelCase      	=		{}
           UpperCAmelCase      	=		add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           UpperCAmelCase      	=		re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )
       @property
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           return len(self.encoder    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           return dict(self.encoder       ,    **self.added_tokens_encoder    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           if token in self.cache:
               return self.cache[token]
           UpperCAmelCase      	=		tuple(lowercase    )
           UpperCAmelCase      	=		get_pairs(lowercase    )

           if not pairs:
               return token

           while True:
               UpperCAmelCase      	=		min(lowercase       ,    key=lambda lowercase    : self.bpe_ranks.get(lowercase       ,    float('''inf'''    )    )    )
               if bigram not in self.bpe_ranks:
                   break
               UpperCAmelCase	,					UpperCAmelCase      	=		bigram
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		0
               while i < len(lowercase    ):
                   try:
                       UpperCAmelCase      	=		word.index(lowercase       ,    lowercase    )
                   except ValueError:
                       new_word.extend(word[i:]    )
                       break
                   else:
                       new_word.extend(word[i:j]    )
                       UpperCAmelCase      	=		j

                   if word[i] == first and i < len(lowercase    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                   else:
                       new_word.append(word[i]    )
                       i += 1
               UpperCAmelCase      	=		tuple(lowercase    )
               UpperCAmelCase      	=		new_word
               if len(lowercase    ) == 1:
                   break
               else:
                   UpperCAmelCase      	=		get_pairs(lowercase    )
           UpperCAmelCase      	=		''' '''.join(lowercase    )
           UpperCAmelCase      	=		word
           return word
       def    A				(     self						:     List[Any]       ,    lowercase						:     int       ,    lowercase						:     Union[str, Any]=None    ):



           '''simple docstring'''

           if self.add_bos_token:
               UpperCAmelCase      	=		[self.bos_token_id]
           else:
               UpperCAmelCase      	=		[]

           UpperCAmelCase      	=		bos_token_ids + token_ids_a

           if token_ids_a is None:
               return output

           return output + bos_token_ids + token_ids_a
       def    A				(     self						:     Dict       ,    lowercase						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for token in re.findall(self.pat       ,    lowercase    ):
               UpperCAmelCase      	=		''''''.join(
                   self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase    ).split(''' '''    )    )
           return bpe_tokens
       def    A				(     self						:     Dict       ,    lowercase						:     int    ):



           '''simple docstring'''

           return self.encoder.get(lowercase       ,    self.encoder.get(self.unk_token    )    )
       def    A				(     self						:     List[str]       ,    lowercase						:     List[Any]    ):



           '''simple docstring'''

           return self.decoder.get(lowercase    )
       def    A				(     self						:     int       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		''''''.join(lowercase    )
           UpperCAmelCase      	=		bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''       ,    errors=self.errors    )
           return text
       def    A				(     self						:     str       ,    lowercase						:     str       ,    lowercase						:     Optional[str] = None    ):



           '''simple docstring'''

           if not os.path.isdir(lowercase    ):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
               return
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as f:
               f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=lowercase       ,    ensure_ascii=lowercase    ) + '''\n'''    )

           UpperCAmelCase      	=		0
           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as writer:
               writer.write('''#version: 0.2\n'''    )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda lowercase    : kv[1]    ):
                   if index != token_index:
                       logger.warning(
                           f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                           ''' Please check that the tokenizer is not corrupted!'''    )
                       UpperCAmelCase      	=		token_index
                   writer.write(''' '''.join(lowercase    ) + '''\n'''    )
                   index += 1

           return vocab_file, merge_file
       def    A				(     self						:     Any       ,    lowercase						:     int       ,    lowercase						:     List[Any]=False       ,    **lowercase						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		kwargs.pop('''add_prefix_space'''       ,    self.add_prefix_space    )
           if is_split_into_words or add_prefix_space:
               UpperCAmelCase      	=		''' ''' + text
           return (text, kwargs)
       def    A				(     self						:     List[Any]       ,    lowercase						:     Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]       ,    lowercase						:     bool = False       ,    lowercase						:     bool = None       ,    lowercase						:     Optional[List[str]] = None       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		super()._decode(
               token_ids=lowercase       ,    skip_special_tokens=lowercase       ,    clean_up_tokenization_spaces=lowercase       ,    **lowercase       ,    )

           if truncate_before_pattern is not None and len(lowercase    ) > 0:
               UpperCAmelCase      	=		self.truncate(lowercase       ,    lowercase    )

           return decoded_text







       def    A				(     self						:     int       ,    lowercase						:     List[Any]       ,    lowercase						:     Union[str, Any]    ):



           '''simple docstring'''

           def find_re(lowercase						:     List[str]       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     Optional[int]    ):
               UpperCAmelCase      	=		pattern.search(lowercase       ,    lowercase    )
               return m.start() if m else -1

           UpperCAmelCase      	=		[re.compile(lowercase       ,    re.MULTILINE    ) for pattern in truncate_before_pattern]

           UpperCAmelCase      	=		list(re.finditer('''^print'''       ,    lowercase       ,    re.MULTILINE    )    )

           if len(lowercase    ) > 1:
               UpperCAmelCase      	=		completion[: prints[1].start()]

           UpperCAmelCase      	=		list(re.finditer('''^def'''       ,    lowercase       ,    re.MULTILINE    )    )

           if len(lowercase    ) > 1:
               UpperCAmelCase      	=		completion[: defs[1].start()]

           UpperCAmelCase      	=		0

           UpperCAmelCase      	=		[
               pos for pos in [find_re(lowercase       ,    lowercase       ,    lowercase    ) for terminal in terminals] if pos != -1
           ]

           if len(lowercase    ) > 0:
               return completion[: min(lowercase    )]
           else:
               return completion










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A			   ={'configuration_dinat': ['DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'DinatConfig']}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A			   =[
       'DINAT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'DinatForImageClassification',
       'DinatModel',
       'DinatPreTrainedModel',
       'DinatBackbone',
   ]

if TYPE_CHECKING:
   from .configuration_dinat import DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP, DinatConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_dinat import (
          DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
          DinatBackbone,
          DinatForImageClassification,
          DinatModel,
          DinatPreTrainedModel,
      )

else:
   import sys

   A			   =_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




