



from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase				:							List[str]								=logging.get_logger(__name__)


class  _lowercase				(a_ ):
    '''simple docstring'''







    lowercase__				  =  ["""input_ids""", """attention_mask"""]

    def __init__(	self       ,	snake_case__="[PAD]"       ,	snake_case__="[BOS]"       ,	snake_case__="[EOS]"       ,	snake_case__="[MASK]"       ,	snake_case__="[CLS]"       ,	snake_case__="[SEP]"       ,	snake_case__=2048       ,	**snake_case__       ,	):

         '''simple docstring'''


         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else pad_token
         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else bos_token
         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else eos_token
         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else mask_token
         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else cls_token
         UpperCamelCase_											=				AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__      ) if isinstance(snake_case__       ,	snake_case__      ) else sep_token

         super().__init__(
             pad_token=snake_case__       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	mask_token=snake_case__       ,	cls_token=snake_case__       ,	sep_token=snake_case__       ,	model_max_length=snake_case__       ,	**snake_case__       ,	)

         UpperCamelCase_											=				2**8  # utf is 8 bits

         # define special tokens dict
         UpperCamelCase_											=				{
             self.pad_token: 0,
             self.bos_token: 1,
             self.eos_token: 2,
             self.mask_token: 3,
             self.cls_token: 4,
             self.sep_token: 5,
         }
         UpperCamelCase_											=				len(self.special_tokens_encoder      )
         UpperCamelCase_											=				{v: k for k, v in self.special_tokens_encoder.items()}

    def  _lowerCamelCase   (	self      ):

         '''simple docstring'''


         UpperCamelCase_											=				self.special_tokens_encoder.copy()
         vocab.update(self.added_tokens_encoder      )
         for i in range(self._utf_vocab_size      ):
              UpperCamelCase_											=				chr(snake_case__      )
              UpperCamelCase_											=				i + len(self.special_tokens_encoder      )
         return vocab

    @property
    def  _lowerCamelCase   (	self      ):

         '''simple docstring'''


         return self._utf_vocab_size + self._num_special_tokens

    def  _lowerCamelCase   (	self       ,	snake_case__       ,	snake_case__ = None       ,	snake_case__ = False      ):

         '''simple docstring'''


         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=snake_case__       ,	token_ids_a=snake_case__       ,	already_has_special_tokens=snake_case__      )

         # normal case: some special tokens
         if token_ids_a is None:
              return [1] + [0] * len(snake_case__      ) + [1]
         return [1] + ([0] * len(snake_case__      )) + [1] + ([0] * len(snake_case__      )) + [1]

    def  _lowerCamelCase   (	self       ,	snake_case__       ,	snake_case__ = None      ):

         '''simple docstring'''


         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         else:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]

    def  _lowerCamelCase   (	self       ,	snake_case__      ):

         '''simple docstring'''


         UpperCamelCase_											=				[chr(snake_case__      ) for i in text.encode("utf-8"      )]
         return tokens

    def  _lowerCamelCase   (	self       ,	snake_case__      ):

         '''simple docstring'''


         if token in self.special_tokens_encoder:
              UpperCamelCase_											=				self.special_tokens_encoder[token]
         elif token in self.added_tokens_encoder:
              UpperCamelCase_											=				self.added_tokens_encoder[token]
         elif len(snake_case__      ) != 1:
              UpperCamelCase_											=				self.unk_token_id
         else:
              UpperCamelCase_											=				ord(snake_case__      ) + self._num_special_tokens
         return token_id

    def  _lowerCamelCase   (	self       ,	snake_case__      ):

         '''simple docstring'''


         if index in self.special_tokens_decoder:
              UpperCamelCase_											=				self.special_tokens_decoder[index]
         elif index in self.added_tokens_decoder:
              UpperCamelCase_											=				self.added_tokens_decoder[index]
         else:
              UpperCamelCase_											=				chr(index - self._num_special_tokens      )
         return token

    def  _lowerCamelCase   (	self       ,	snake_case__      ):

         '''simple docstring'''


         UpperCamelCase_											=				b""
         for token in tokens:
              if token in self.special_tokens_decoder:
                   UpperCamelCase_											=				self.special_tokens_decoder[token].encode("utf-8"      )
              elif token in self.added_tokens_decoder:
                   UpperCamelCase_											=				self.special_tokens_decoder[token].encode("utf-8"      )
              elif token in self.special_tokens_encoder:
                   UpperCamelCase_											=				token.encode("utf-8"      )
              elif token in self.added_tokens_encoder:
                   UpperCamelCase_											=				token.encode("utf-8"      )
              else:
                   UpperCamelCase_											=				bytes([ord(snake_case__      )]      )
              bstring += tok_string
         UpperCamelCase_											=				bstring.decode("utf-8"       ,	errors="replace"      )
         return string

    def  _lowerCamelCase   (	self       ,	snake_case__       ,	snake_case__ = None      ):

         '''simple docstring'''


         return ()





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCAmelCase				:							Optional[int]								={
    """configuration_realm""": ["""REALM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RealmConfig"""],
    """tokenization_realm""": ["""RealmTokenizer"""],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase				:							Optional[int]								=["""RealmTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCAmelCase				:							Union[str, Any]								=[
	    """REALM_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """RealmEmbedder""",
	    """RealmForOpenQA""",
	    """RealmKnowledgeAugEncoder""",
	    """RealmPreTrainedModel""",
	    """RealmReader""",
	    """RealmScorer""",
	    """load_tf_weights_in_realm""",
	]
	UpperCAmelCase				:							List[str]								=["""RealmRetriever"""]


if TYPE_CHECKING:
	from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
	from .tokenization_realm import RealmTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_realm import RealmTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_realm import (
		    REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    RealmEmbedder,
		    RealmForOpenQA,
		    RealmKnowledgeAugEncoder,
		    RealmPreTrainedModel,
		    RealmReader,
		    RealmScorer,
		    load_tf_weights_in_realm,
		)
		from .retrieval_realm import RealmRetriever


else:
	import sys

	UpperCAmelCase				:							str								=_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
