



'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


A			:   str			=      logging.get_logger(__name__)

A			:   Optional[int]			=      {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

A			:   List[str]			=      {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

A			:   Tuple			=      {
    """Salesforce/codegen-350M-mono""": 2_048,
}
@lru_cache()
def 		_a   (  ):
   snake_case   :							Any						  =(
       list(range(ord('''!'''							)					,						ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)					,						ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)					,						ord('''ÿ'''							) + 1							)							)
   )
   snake_case   :							Dict						  =bs[:]
   snake_case   :							int						  =0
   for b in range(2**8							):
      if b not in bs:
         bs.append(lowerCamelCase_							)
         cs.append(2**8 + n							)
         n += 1
   snake_case   :							Optional[Any]						  =[chr(lowerCamelCase_							) for n in cs]
   return dict(zip(lowerCamelCase_					,						lowerCamelCase_							)							)
def 		_a   (  lowerCamelCase_							):
   snake_case   :							Union[str, Any]						  =set()
   snake_case   :							Tuple						  =word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)							)
      snake_case   :							int						  =char
   return pairs





class 							lowerCAmelCase_		(	a_						):
    __UpperCAmelCase							=	VOCAB_FILES_NAMES
    __UpperCAmelCase							=	PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase							=	['input_ids', 'attention_mask']





    def __init__(    self    :    Dict,    _snake_case    :    Dict,    _snake_case    :    Optional[Any],    _snake_case    :    int="replace",    _snake_case    :    Optional[int]="<|endoftext|>",    _snake_case    :    Dict="<|endoftext|>",    _snake_case    :    Optional[Any]="<|endoftext|>",    _snake_case    :    Union[str, Any]=None,    _snake_case    :    List[str]=False,    _snake_case    :    List[Any]=False,    **_snake_case    :    Any,    ):



       '''simple docstring'''







       snake_case   :							Any						  =AddedToken(_snake_case,    lstrip=_snake_case,    rstrip=_snake_case     ) if isinstance(_snake_case,    _snake_case     ) else bos_token
       snake_case   :							int						  =AddedToken(_snake_case,    lstrip=_snake_case,    rstrip=_snake_case     ) if isinstance(_snake_case,    _snake_case     ) else eos_token
       snake_case   :							Tuple						  =AddedToken(_snake_case,    lstrip=_snake_case,    rstrip=_snake_case     ) if isinstance(_snake_case,    _snake_case     ) else unk_token
       snake_case   :							Optional[Any]						  =AddedToken(_snake_case,    lstrip=_snake_case,    rstrip=_snake_case     ) if isinstance(_snake_case,    _snake_case     ) else pad_token
       super().__init__(
           errors=_snake_case,    unk_token=_snake_case,    bos_token=_snake_case,    eos_token=_snake_case,    pad_token=_snake_case,    add_prefix_space=_snake_case,    add_bos_token=_snake_case,    **_snake_case,    )
       snake_case   :							int						  =add_bos_token

       with open(_snake_case,    encoding='''utf-8'''     ) as vocab_handle:
          snake_case   :							Dict						  =json.load(_snake_case     )
       snake_case   :							Union[str, Any]						  ={v: k for k, v in self.encoder.items()}
       snake_case   :							Optional[Any]						  =errors  # how to handle errors in decoding
       snake_case   :							Union[str, Any]						  =bytes_to_unicode()
       snake_case   :							Union[str, Any]						  ={v: k for k, v in self.byte_encoder.items()}
       with open(_snake_case,    encoding='''utf-8'''     ) as merges_handle:
          snake_case   :							List[Any]						  =merges_handle.read().split('''\n'''     )[1:-1]
       snake_case   :							Optional[Any]						  =[tuple(merge.split()     ) for merge in bpe_merges]
       snake_case   :							Optional[Any]						  =dict(zip(_snake_case,    range(len(_snake_case     )     )     )     )
       snake_case   :							Optional[Any]						  ={}
       snake_case   :							Union[str, Any]						  =add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       snake_case   :							int						  =re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''     )





    @property
    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       return len(self.encoder     )





    def 					__snake_case       (    self    :    Union[str, Any]     ):



       '''simple docstring'''







       return dict(self.encoder,    **self.added_tokens_encoder     )





    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       if token in self.cache:
          return self.cache[token]
       snake_case   :							Optional[int]						  =tuple(_snake_case     )
       snake_case   :							Optional[Any]						  =get_pairs(_snake_case     )

       if not pairs:
          return token

       while True:
          snake_case   :							Union[str, Any]						  =min(_snake_case,    key=lambda _snake_case     : self.bpe_ranks.get(_snake_case,    float('''inf'''     )     )     )
          if bigram not in self.bpe_ranks:
             break
          snake_case   :							Tuple						  =bigram
          snake_case   :							Dict						  =[]
          snake_case   :							List[Any]						  =0
          while i < len(_snake_case     ):
             try:
                snake_case   :							Optional[int]						  =word.index(_snake_case,    _snake_case     )
             except ValueError:
                new_word.extend(word[i:]     )
                break
             else:
                new_word.extend(word[i:j]     )
                snake_case   :							Optional[Any]						  =j

             if word[i] == first and i < len(_snake_case     ) - 1 and word[i + 1] == second:
                new_word.append(first + second     )
                i += 2
             else:
                new_word.append(word[i]     )
                i += 1
          snake_case   :							List[str]						  =tuple(_snake_case     )
          snake_case   :							Dict						  =new_word
          if len(_snake_case     ) == 1:
             break
          else:
             snake_case   :							int						  =get_pairs(_snake_case     )
       snake_case   :							int						  =''' '''.join(_snake_case     )
       snake_case   :							Union[str, Any]						  =word
       return word





    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    Union[str, Any],    _snake_case    :    Any=None     ):



       '''simple docstring'''







       if self.add_bos_token:
          snake_case   :							str						  =[self.bos_token_id]
       else:
          snake_case   :							Union[str, Any]						  =[]

       snake_case   :							List[str]						  =bos_token_ids + token_ids_a

       if token_ids_a is None:
          return output

       return output + bos_token_ids + token_ids_a





    def 					__snake_case       (    self    :    int,    _snake_case    :    List[Any]     ):



       '''simple docstring'''







       snake_case   :							Tuple						  =[]
       for token in re.findall(self.pat,    _snake_case     ):
          snake_case   :							List[str]						  =''''''.join(
              self.byte_encoder[b] for b in token.encode('''utf-8'''     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case     ).split(''' '''     )     )
       return bpe_tokens





    def 					__snake_case       (    self    :    str,    _snake_case    :    Tuple     ):



       '''simple docstring'''







       return self.encoder.get(_snake_case,    self.encoder.get(self.unk_token     )     )





    def 					__snake_case       (    self    :    str,    _snake_case    :    Optional[Any]     ):



       '''simple docstring'''







       return self.decoder.get(_snake_case     )





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							Dict						  =''''''.join(_snake_case     )
       snake_case   :							Tuple						  =bytearray([self.byte_decoder[c] for c in text]     ).decode('''utf-8''',    errors=self.errors     )
       return text





    def 					__snake_case       (    self    :    Any,    _snake_case    :    str,    _snake_case    :    Optional[str] = None     ):



       '''simple docstring'''







       if not os.path.isdir(_snake_case     ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
          return
       snake_case   :							int						  =os.path.join(
           _snake_case,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
       snake_case   :							Optional[Any]						  =os.path.join(
           _snake_case,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

       with open(_snake_case,    '''w''',    encoding='''utf-8'''     ) as f:
          f.write(json.dumps(self.encoder,    indent=2,    sort_keys=_snake_case,    ensure_ascii=_snake_case     ) + '''\n'''     )

       snake_case   :							str						  =0
       with open(_snake_case,    '''w''',    encoding='''utf-8'''     ) as writer:
          writer.write('''#version: 0.2\n'''     )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),    key=lambda _snake_case     : kv[1]     ):
             if index != token_index:
                logger.warning(
                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                    ''' Please check that the tokenizer is not corrupted!'''     )
                snake_case   :							List[str]						  =token_index
             writer.write(''' '''.join(_snake_case     ) + '''\n'''     )
             index += 1

       return vocab_file, merge_file





    def 					__snake_case       (    self    :    str,    _snake_case    :    str,    _snake_case    :    List[Any]=False,    **_snake_case    :    Union[str, Any]     ):



       '''simple docstring'''







       snake_case   :							int						  =kwargs.pop('''add_prefix_space''',    self.add_prefix_space     )
       if is_split_into_words or add_prefix_space:
          snake_case   :							Union[str, Any]						  =''' ''' + text
       return (text, kwargs)





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],    _snake_case    :    bool = False,    _snake_case    :    bool = None,    _snake_case    :    Optional[List[str]] = None,    **_snake_case    :    Dict,    ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =super()._decode(
           token_ids=_snake_case,    skip_special_tokens=_snake_case,    clean_up_tokenization_spaces=_snake_case,    **_snake_case,    )

       if truncate_before_pattern is not None and len(_snake_case     ) > 0:
          snake_case   :							Union[str, Any]						  =self.truncate(_snake_case,    _snake_case     )

       return decoded_text






    def 					__snake_case       (    self    :    Optional[int],    _snake_case    :    str,    _snake_case    :    Tuple     ):



       '''simple docstring'''







       def find_re(_snake_case    :    str,    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any]     ):
          snake_case   :							Union[str, Any]						  =pattern.search(_snake_case,    _snake_case     )
          return m.start() if m else -1

       snake_case   :							Any						  =[re.compile(_snake_case,    re.MULTILINE     ) for pattern in truncate_before_pattern]

       snake_case   :							Tuple						  =list(re.finditer('''^print''',    _snake_case,    re.MULTILINE     )     )

       if len(_snake_case     ) > 1:
          snake_case   :							Tuple						  =completion[: prints[1].start()]

       snake_case   :							int						  =list(re.finditer('''^def''',    _snake_case,    re.MULTILINE     )     )

       if len(_snake_case     ) > 1:
          snake_case   :							Any						  =completion[: defs[1].start()]

       snake_case   :							int						  =0

       snake_case   :							Union[str, Any]						  =[
           pos for pos in [find_re(_snake_case,    _snake_case,    _snake_case     ) for terminal in terminals] if pos != -1
       ]

       if len(_snake_case     ) > 0:
          return completion[: min(_snake_case     )]
       else:
          return completion






'''simple docstring'''





import doctest

import projectq
from projectq.ops import H, Measure
def 		_a   (  lowerCamelCase_							):
   snake_case   :							str						  =quantum_engine.allocate_qubit()
   H | qubit
   Measure | qubit
   return int(lowerCamelCase_							)


if __name__ == "__main__":
      doctest.testmod()

      # initialises a new quantum backend
      A			:   Dict			=      projectq.MainEngine()

      # Generate a list of 10 random numbers
      A			:   Dict			=      [get_random_number(quantum_engine) for _ in range(10)]

      # Flushes the quantum engine from memory
      quantum_engine.flush()

      print("""Random numbers""", random_numbers_list)





