
'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


__A       =      logging.get_logger(__name__)

__A       =      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A       =      {
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A       =      {
    "microsoft/layoutlm-base-uncased": 512,
    "microsoft/layoutlm-large-uncased": 512,
}

__A       =      {
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}





class      A							(    __UpperCAmelCase			):
			lowerCamelCase							:							int    						=					VOCAB_FILES_NAMES
			lowerCamelCase							:							int    						=					PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase							:							List[Any]    						=					PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase							:							Any    						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase							:							Tuple    						=					LayoutLMTokenizer



			def __init__(   self    ,       lowerCamelCase__=None    ,       lowerCamelCase__=None    ,       lowerCamelCase__=True    ,       lowerCamelCase__="[UNK]"    ,       lowerCamelCase__="[SEP]"    ,       lowerCamelCase__="[PAD]"    ,       lowerCamelCase__="[CLS]"    ,       lowerCamelCase__="[MASK]"    ,       lowerCamelCase__=True    ,       lowerCamelCase__=None    ,       **lowerCamelCase__    ,       )     ->						Optional[Any]:




									'''simple docstring'''



									super().__init__(
									    lowerCamelCase__    ,       tokenizer_file=lowerCamelCase__    ,       do_lower_case=lowerCamelCase__    ,       unk_token=lowerCamelCase__    ,       sep_token=lowerCamelCase__    ,       pad_token=lowerCamelCase__    ,       cls_token=lowerCamelCase__    ,       mask_token=lowerCamelCase__    ,       tokenize_chinese_chars=lowerCamelCase__    ,       strip_accents=lowerCamelCase__    ,       **lowerCamelCase__    ,       )

									lowercase__								=    json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
									if (
									    normalizer_state.get("""lowercase"""    ,       lowerCamelCase__    ) != do_lower_case
									    or normalizer_state.get("""strip_accents"""    ,       lowerCamelCase__    ) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars"""    ,       lowerCamelCase__    ) != tokenize_chinese_chars
									):
															lowercase__								=    getattr(lowerCamelCase__    ,       normalizer_state.pop("""type"""    )    )
															lowercase__								=    do_lower_case
															lowercase__								=    strip_accents
															lowercase__								=    tokenize_chinese_chars
															lowercase__								=    normalizer_class(**lowerCamelCase__    )

									lowercase__								=    do_lower_case



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__=None    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__ = None    )     ->						List[int]:




									'''simple docstring'''



									lowercase__								=    [self.sep_token_id]
									lowercase__								=    [self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
									return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__ = None    )     ->						Tuple[str]:




									'''simple docstring'''



									lowercase__								=    self._tokenizer.model.save(lowerCamelCase__    ,       name=lowerCamelCase__    )
									return tuple(lowerCamelCase__    )








'''simple docstring'''



import warnings

from .generation import FlaxGenerationMixin





class      A							(    __UpperCAmelCase			):
			# warning at import time
			warnings.warn(
			    """Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will """
			    """be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead."""						,  __UpperCAmelCase						,  )






