

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


lowerCAmelCase       :				Dict         =					logging.get_logger(__name__)

lowerCAmelCase       :				List[str]         =					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase       :				Tuple         =					{
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase       :				int         =					{
    'microsoft/layoutlm-base-uncased': 5_12,
    'microsoft/layoutlm-large-uncased': 5_12,
}

lowerCAmelCase       :				Dict         =					{
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}
class 	_A						(					__magic_name__):
  SCREAMING_SNAKE_CASE      : Any    =					VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE      : Union[str, Any]    =					PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE      : int    =					PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE      : str    =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE      : Union[str, Any]    =					LayoutLMTokenizer

  def __init__(			self	,   _SCREAMING_SNAKE_CASE=None	,   _SCREAMING_SNAKE_CASE=None	,   _SCREAMING_SNAKE_CASE=True	,   _SCREAMING_SNAKE_CASE="[UNK]"	,   _SCREAMING_SNAKE_CASE="[SEP]"	,   _SCREAMING_SNAKE_CASE="[PAD]"	,   _SCREAMING_SNAKE_CASE="[CLS]"	,   _SCREAMING_SNAKE_CASE="[MASK]"	,   _SCREAMING_SNAKE_CASE=True	,   _SCREAMING_SNAKE_CASE=None	,   **_SCREAMING_SNAKE_CASE	,   ):





        """simple docstring"""







        super().__init__(
            _SCREAMING_SNAKE_CASE	,   tokenizer_file=_SCREAMING_SNAKE_CASE	,   do_lower_case=_SCREAMING_SNAKE_CASE	,   unk_token=_SCREAMING_SNAKE_CASE	,   sep_token=_SCREAMING_SNAKE_CASE	,   pad_token=_SCREAMING_SNAKE_CASE	,   cls_token=_SCREAMING_SNAKE_CASE	,   mask_token=_SCREAMING_SNAKE_CASE	,   tokenize_chinese_chars=_SCREAMING_SNAKE_CASE	,   strip_accents=_SCREAMING_SNAKE_CASE	,   **_SCREAMING_SNAKE_CASE	,   )

        SCREAMING_SNAKE_CASE_			:					int          =       json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
        if (
            normalizer_state.get('lowercase'	,   _SCREAMING_SNAKE_CASE   ) != do_lower_case
            or normalizer_state.get('strip_accents'	,   _SCREAMING_SNAKE_CASE   ) != strip_accents
            or normalizer_state.get('handle_chinese_chars'	,   _SCREAMING_SNAKE_CASE   ) != tokenize_chinese_chars
        ):
              SCREAMING_SNAKE_CASE_			:					Any          =       getattr(_SCREAMING_SNAKE_CASE	,   normalizer_state.pop('type'   )   )
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       do_lower_case
              SCREAMING_SNAKE_CASE_			:					Any          =       strip_accents
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       tokenize_chinese_chars
              SCREAMING_SNAKE_CASE_			:					Optional[Any]          =       normalizer_class(**_SCREAMING_SNAKE_CASE   )

        SCREAMING_SNAKE_CASE_			:					Union[str, Any]          =       do_lower_case

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE=None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					Optional[int]          =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output

  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					str          =       [self.sep_token_id]
        SCREAMING_SNAKE_CASE_			:					List[str]          =       [self.cls_token_id]
        if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
        return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



  def 	UpperCAmelCase      (			self	,   _SCREAMING_SNAKE_CASE	,   _SCREAMING_SNAKE_CASE = None   ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_			:					List[str]          =       self._tokenizer.model.save(_SCREAMING_SNAKE_CASE	,   name=_SCREAMING_SNAKE_CASE   )
        return tuple(_SCREAMING_SNAKE_CASE   )









from typing import TYPE_CHECKING

from ...utils import _LazyModule


lowerCAmelCase       :				Tuple         =					{'tokenization_phobert': ['PhobertTokenizer']}


if TYPE_CHECKING:
    from .tokenization_phobert import PhobertTokenizer

else:
    import sys

    lowerCAmelCase       :				Optional[int]         =					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






