'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowercase				   =  logging.get_logger(__name__)

__lowercase				   =  {
    'flaubert/flaubert_small_cased': 'https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json',
    'flaubert/flaubert_base_uncased': 'https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json',
    'flaubert/flaubert_base_cased': 'https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json',
    'flaubert/flaubert_large_cased': 'https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json',
}






class     a__( _A     ):


   '''simple docstring'''






   UpperCAmelCase_      :					str								=	'''flaubert'''
   UpperCAmelCase_      :					Optional[Any]								=	{
       '''hidden_size''': '''emb_dim''',
       '''num_attention_heads''': '''n_heads''',
       '''num_hidden_layers''': '''n_layers''',
       '''n_words''': '''vocab_size''',  # For backward compatibility
   }



   def __init__(       self	,   __lowerCAmelCase=False	,   __lowerCAmelCase=0.0	,   __lowerCAmelCase=30145	,   __lowerCAmelCase=2048	,   __lowerCAmelCase=12	,   __lowerCAmelCase=16	,   __lowerCAmelCase=0.1	,   __lowerCAmelCase=0.1	,   __lowerCAmelCase=True	,   __lowerCAmelCase=False	,   __lowerCAmelCase=False	,   __lowerCAmelCase=False	,   __lowerCAmelCase=1	,   __lowerCAmelCase=True	,   __lowerCAmelCase=512	,   __lowerCAmelCase=2048**-0.5	,   __lowerCAmelCase=1E-1_2	,   __lowerCAmelCase=0.02	,   __lowerCAmelCase=0	,   __lowerCAmelCase=1	,   __lowerCAmelCase=2	,   __lowerCAmelCase=3	,   __lowerCAmelCase=5	,   __lowerCAmelCase=True	,   __lowerCAmelCase="first"	,   __lowerCAmelCase=True	,   __lowerCAmelCase=None	,   __lowerCAmelCase=True	,   __lowerCAmelCase=0.1	,   __lowerCAmelCase=5	,   __lowerCAmelCase=5	,   __lowerCAmelCase=0	,   __lowerCAmelCase=0	,   __lowerCAmelCase=2	,   __lowerCAmelCase=0	,   **__lowerCAmelCase	,   ):




     """simple docstring"""


     lowerCAmelCase        =  pre_norm
     lowerCAmelCase        =  layerdrop
     lowerCAmelCase        =  vocab_size
     lowerCAmelCase        =  emb_dim
     lowerCAmelCase        =  n_layers
     lowerCAmelCase        =  n_heads
     lowerCAmelCase        =  dropout
     lowerCAmelCase        =  attention_dropout
     lowerCAmelCase        =  gelu_activation
     lowerCAmelCase        =  sinusoidal_embeddings
     lowerCAmelCase        =  causal
     lowerCAmelCase        =  asm
     lowerCAmelCase        =  n_langs
     lowerCAmelCase        =  use_lang_emb
     lowerCAmelCase        =  layer_norm_eps
     lowerCAmelCase        =  bos_index
     lowerCAmelCase        =  eos_index
     lowerCAmelCase        =  pad_index
     lowerCAmelCase        =  unk_index
     lowerCAmelCase        =  mask_index
     lowerCAmelCase        =  is_encoder
     lowerCAmelCase        =  max_position_embeddings
     lowerCAmelCase        =  embed_init_std
     lowerCAmelCase        =  init_std
     lowerCAmelCase        =  summary_type
     lowerCAmelCase        =  summary_use_proj
     lowerCAmelCase        =  summary_activation
     lowerCAmelCase        =  summary_proj_to_labels
     lowerCAmelCase        =  summary_first_dropout
     lowerCAmelCase        =  start_n_top
     lowerCAmelCase        =  end_n_top
     lowerCAmelCase        =  mask_token_id
     lowerCAmelCase        =  lang_id

     if "n_words" in kwargs:
       lowerCAmelCase        =  kwargs["""n_words"""]

     super().__init__(pad_token_id=UpperCamelCase__	,   bos_token_id=UpperCamelCase__	,   **UpperCamelCase__)







class     a__( _A     ):


   '''simple docstring'''






   @property
   def 							a_	(       self):




     """simple docstring"""


     if self.task == "multiple-choice":
       lowerCAmelCase        =  {0: """batch""", 1: """choice""", 2: """sequence"""}
     else:
       lowerCAmelCase        =  {0: """batch""", 1: """sequence"""}
     return OrderedDict(
         [
             ("""input_ids""", dynamic_axis),
             ("""attention_mask""", dynamic_axis),
         ])





'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowercase				   =  logging.get_logger(__name__)

__lowercase				   =  {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowercase				   =  {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

__lowercase				   =  {
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}






@lru_cache()
def     snake_case__							(      )   ->     int:

		'''simple docstring'''


		lowerCAmelCase        =  (
		    list(range(ord("""!"""			)				,       ord("""~"""			) + 1			)			) + list(range(ord("""¡"""			)				,       ord("""¬"""			) + 1			)			) + list(range(ord("""®"""			)				,       ord("""ÿ"""			) + 1			)			)
		)
		lowerCAmelCase        =  bs[:]
		lowerCAmelCase        =  0
		for b in range(2**8			):
				if b not in bs:
						bs.append(_A			)
						cs.append(2**8 + n			)
						n += 1
		lowerCAmelCase        =  [chr(_A			) for n in cs]
		return dict(zip(_A				,       _A			)			)






def     snake_case__							(      _A: int			)   ->     int:

		'''simple docstring'''


		lowerCAmelCase        =  set()
		lowerCAmelCase        =  word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)			)
				lowerCAmelCase        =  char
		return pairs






class     a__( lowerCAmelCase__     ):


			'''simple docstring'''






			UpperCAmelCase_      :					Tuple								=	VOCAB_FILES_NAMES
			UpperCAmelCase_      :					List[str]								=	PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_      :					Any								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_      :					str								=	['''input_ids''', '''attention_mask''']



			def __init__(       self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase="replace"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase="<|endoftext|>"	,   __lowerCAmelCase=None	,   __lowerCAmelCase=False	,   __lowerCAmelCase=False	,   **__lowerCAmelCase	,   ):




					"""simple docstring"""


					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else bos_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else eos_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else unk_token
					lowerCAmelCase        =  AddedToken(__lowerCAmelCase	,   lstrip=__lowerCAmelCase	,   rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase	,   __lowerCAmelCase) else pad_token
					super().__init__(
					    errors=__lowerCAmelCase	,   unk_token=__lowerCAmelCase	,   bos_token=__lowerCAmelCase	,   eos_token=__lowerCAmelCase	,   pad_token=__lowerCAmelCase	,   add_prefix_space=__lowerCAmelCase	,   add_bos_token=__lowerCAmelCase	,   **__lowerCAmelCase	,   )
					lowerCAmelCase        =  add_bos_token

					with open(__lowerCAmelCase	,   encoding="""utf-8""") as vocab_handle:
							lowerCAmelCase        =  json.load(__lowerCAmelCase)
					lowerCAmelCase        =  {v: k for k, v in self.encoder.items()}
					lowerCAmelCase        =  errors  # how to handle errors in decoding
					lowerCAmelCase        =  bytes_to_unicode()
					lowerCAmelCase        =  {v: k for k, v in self.byte_encoder.items()}
					with open(__lowerCAmelCase	,   encoding="""utf-8""") as merges_handle:
							lowerCAmelCase        =  merges_handle.read().split("""\n""")[1:-1]
					lowerCAmelCase        =  [tuple(merge.split()) for merge in bpe_merges]
					lowerCAmelCase        =  dict(zip(__lowerCAmelCase	,   range(len(__lowerCAmelCase))))
					lowerCAmelCase        =  {}
					lowerCAmelCase        =  add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					lowerCAmelCase        =  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")



			@property
			def 							a_	(       self):




					"""simple docstring"""


					return len(self.encoder)



			def 							a_	(       self):




					"""simple docstring"""


					return dict(self.encoder	,   **self.added_tokens_encoder)



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					if token in self.cache:
							return self.cache[token]
					lowerCAmelCase        =  tuple(__lowerCAmelCase)
					lowerCAmelCase        =  get_pairs(__lowerCAmelCase)

					if not pairs:
							return token

					while True:
							lowerCAmelCase        =  min(__lowerCAmelCase	,   key=lambda __lowerCAmelCase: self.bpe_ranks.get(__lowerCAmelCase	,   float("""inf""")))
							if bigram not in self.bpe_ranks:
									break
							lowerCAmelCase ,					lowerCAmelCase        =  bigram
							lowerCAmelCase        =  []
							lowerCAmelCase        =  0
							while i < len(__lowerCAmelCase):
									try:
											lowerCAmelCase        =  word.index(__lowerCAmelCase	,   __lowerCAmelCase)
									except ValueError:
											new_word.extend(word[i:])
											break
									else:
											new_word.extend(word[i:j])
											lowerCAmelCase        =  j

									if word[i] == first and i < len(__lowerCAmelCase) - 1 and word[i + 1] == second:
											new_word.append(first + second)
											i += 2
									else:
											new_word.append(word[i])
											i += 1
							lowerCAmelCase        =  tuple(__lowerCAmelCase)
							lowerCAmelCase        =  new_word
							if len(__lowerCAmelCase) == 1:
									break
							else:
									lowerCAmelCase        =  get_pairs(__lowerCAmelCase)
					lowerCAmelCase        =  """ """.join(__lowerCAmelCase)
					lowerCAmelCase        =  word
					return word



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase=None):




					"""simple docstring"""


					if self.add_bos_token:
							lowerCAmelCase        =  [self.bos_token_id]
					else:
							lowerCAmelCase        =  []

					lowerCAmelCase        =  bos_token_ids + token_ids_a

					if token_ids_a is None:
							return output

					return output + bos_token_ids + token_ids_a



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  []
					for token in re.findall(self.pat	,   __lowerCAmelCase):
							lowerCAmelCase        =  """""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase).split(""" """))
					return bpe_tokens



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					return self.encoder.get(__lowerCAmelCase	,   self.encoder.get(self.unk_token))



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					return self.decoder.get(__lowerCAmelCase)



			def 							a_	(       self	,   __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  """""".join(__lowerCAmelCase)
					lowerCAmelCase        =  bytearray([self.byte_decoder[c] for c in text]).decode("""utf-8"""	,   errors=self.errors)
					return text



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase = None):




					"""simple docstring"""


					if not os.path.isdir(__lowerCAmelCase):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory")
							return
					lowerCAmelCase        =  os.path.join(
					    __lowerCAmelCase	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
					lowerCAmelCase        =  os.path.join(
					    __lowerCAmelCase	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

					with open(__lowerCAmelCase	,   """w"""	,   encoding="""utf-8""") as f:
							f.write(json.dumps(self.encoder	,   indent=2	,   sort_keys=__lowerCAmelCase	,   ensure_ascii=__lowerCAmelCase) + """\n""")

					lowerCAmelCase        =  0
					with open(__lowerCAmelCase	,   """w"""	,   encoding="""utf-8""") as writer:
							writer.write("""#version: 0.2\n""")
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,   key=lambda __lowerCAmelCase: kv[1]):
									if index != token_index:
											logger.warning(
											    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
											    """ Please check that the tokenizer is not corrupted!""")
											lowerCAmelCase        =  token_index
									writer.write(""" """.join(__lowerCAmelCase) + """\n""")
									index += 1

					return vocab_file, merge_file



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase=False	,   **__lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase        =  kwargs.pop("""add_prefix_space"""	,   self.add_prefix_space)
					if is_split_into_words or add_prefix_space:
							lowerCAmelCase        =  """ """ + text
					return (text, kwargs)



			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase = False	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   ):




					"""simple docstring"""


					lowerCAmelCase        =  super()._decode(
					    token_ids=__lowerCAmelCase	,   skip_special_tokens=__lowerCAmelCase	,   clean_up_tokenization_spaces=__lowerCAmelCase	,   **__lowerCAmelCase	,   )

					if truncate_before_pattern is not None and len(__lowerCAmelCase) > 0:
							lowerCAmelCase        =  self.truncate(__lowerCAmelCase	,   __lowerCAmelCase)

					return decoded_text







			def 							a_	(       self	,   __lowerCAmelCase	,   __lowerCAmelCase):




					"""simple docstring"""


					def find_re(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase):
							lowerCAmelCase        =  pattern.search(__lowerCAmelCase	,   __lowerCAmelCase)
							return m.start() if m else -1

					lowerCAmelCase        =  [re.compile(__lowerCAmelCase	,   re.MULTILINE) for pattern in truncate_before_pattern]

					lowerCAmelCase        =  list(re.finditer("""^print"""	,   __lowerCAmelCase	,   re.MULTILINE))

					if len(__lowerCAmelCase) > 1:
							lowerCAmelCase        =  completion[: prints[1].start()]

					lowerCAmelCase        =  list(re.finditer("""^def"""	,   __lowerCAmelCase	,   re.MULTILINE))

					if len(__lowerCAmelCase) > 1:
							lowerCAmelCase        =  completion[: defs[1].start()]

					lowerCAmelCase        =  0

					lowerCAmelCase        =  [
					    pos for pos in [find_re(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase) for terminal in terminals] if pos != -1
					]

					if len(__lowerCAmelCase) > 0:
							return completion[: min(__lowerCAmelCase)]
					else:
							return completion




