




"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


__A							=     logging.get_logger(__name__)

__A							=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A							=     {
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A							=     {
    "microsoft/layoutlm-base-uncased": 512,
    "microsoft/layoutlm-large-uncased": 512,
}

__A							=     {
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}
class      lowerCamelCase__      (					lowerCamelCase_     ):
			a__	: List[Any]        =       VOCAB_FILES_NAMES
			a__	: int        =       PRETRAINED_VOCAB_FILES_MAP
			a__	: Any        =       PRETRAINED_INIT_CONFIGURATION
			a__	: str        =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a__	: int        =       LayoutLMTokenizer


			def __init__(       self			,	SCREAMING_SNAKE_CASE=None			,	SCREAMING_SNAKE_CASE=None			,	SCREAMING_SNAKE_CASE=True			,	SCREAMING_SNAKE_CASE="[UNK]"			,	SCREAMING_SNAKE_CASE="[SEP]"			,	SCREAMING_SNAKE_CASE="[PAD]"			,	SCREAMING_SNAKE_CASE="[CLS]"			,	SCREAMING_SNAKE_CASE="[MASK]"			,	SCREAMING_SNAKE_CASE=True			,	SCREAMING_SNAKE_CASE=None			,	**SCREAMING_SNAKE_CASE			,	):



					"""simple docstring"""







					super().__init__(
					    SCREAMING_SNAKE_CASE			,	tokenizer_file=SCREAMING_SNAKE_CASE			,	do_lower_case=SCREAMING_SNAKE_CASE			,	unk_token=SCREAMING_SNAKE_CASE			,	sep_token=SCREAMING_SNAKE_CASE			,	pad_token=SCREAMING_SNAKE_CASE			,	cls_token=SCREAMING_SNAKE_CASE			,	mask_token=SCREAMING_SNAKE_CASE			,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE			,	strip_accents=SCREAMING_SNAKE_CASE			,	**SCREAMING_SNAKE_CASE			,	)

					snake_case						:  Tuple        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
					if (
					    normalizer_state.get("lowercase"			,	SCREAMING_SNAKE_CASE							) != do_lower_case
					    or normalizer_state.get("strip_accents"			,	SCREAMING_SNAKE_CASE							) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"			,	SCREAMING_SNAKE_CASE							) != tokenize_chinese_chars
					):
							snake_case						:  str        =			getattr(SCREAMING_SNAKE_CASE			,	normalizer_state.pop("type"							)							)
							snake_case						:  Tuple        =			do_lower_case
							snake_case						:  List[str]        =			strip_accents
							snake_case						:  str        =			tokenize_chinese_chars
							snake_case						:  Tuple        =			normalizer_class(**SCREAMING_SNAKE_CASE							)

					snake_case						:  Tuple        =			do_lower_case


			def 							lowerCamelCase_			(       self			,	SCREAMING_SNAKE_CASE			,	SCREAMING_SNAKE_CASE=None							):



					"""simple docstring"""







					snake_case						:  Optional[Any]        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output


			def 							lowerCamelCase_			(       self			,	SCREAMING_SNAKE_CASE			,	SCREAMING_SNAKE_CASE = None							):



					"""simple docstring"""







					snake_case						:  Optional[Any]        =			[self.sep_token_id]
					snake_case						:  Any        =			[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep							) * [0]
					return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]



			def 							lowerCamelCase_			(       self			,	SCREAMING_SNAKE_CASE			,	SCREAMING_SNAKE_CASE = None							):



					"""simple docstring"""







					snake_case						:  List[Any]        =			self._tokenizer.model.save(SCREAMING_SNAKE_CASE			,	name=SCREAMING_SNAKE_CASE							)
					return tuple(SCREAMING_SNAKE_CASE							)







"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


__A							=     {"configuration_gptj": ["GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTJConfig", "GPTJOnnxConfig"]}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__A							=     [
							    "GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "GPTJForCausalLM",
							    "GPTJForQuestionAnswering",
							    "GPTJForSequenceClassification",
							    "GPTJModel",
							    "GPTJPreTrainedModel",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__A							=     [
							    "TFGPTJForCausalLM",
							    "TFGPTJForQuestionAnswering",
							    "TFGPTJForSequenceClassification",
							    "TFGPTJModel",
							    "TFGPTJPreTrainedModel",
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__A							=     [
							    "FlaxGPTJForCausalLM",
							    "FlaxGPTJModel",
							    "FlaxGPTJPreTrainedModel",
							]


if TYPE_CHECKING:
							from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_gptj import (
														    GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
														    GPTJForCausalLM,
														    GPTJForQuestionAnswering,
														    GPTJForSequenceClassification,
														    GPTJModel,
														    GPTJPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_gptj import (
														    TFGPTJForCausalLM,
														    TFGPTJForQuestionAnswering,
														    TFGPTJForSequenceClassification,
														    TFGPTJModel,
														    TFGPTJPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
							import sys

							__A							=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

