





'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


snake_case_       :    int          =					logging.get_logger(__name__)

snake_case_       :    Dict          =					{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

snake_case_       :    Optional[Any]          =					{
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

snake_case_       :    Any          =					{
    "microsoft/layoutlm-base-uncased": 5_12,
    "microsoft/layoutlm-large-uncased": 5_12,
}

snake_case_       :    Tuple          =					{
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}


class    __a (lowerCamelCase       ):
							__a   :				Tuple       =    VOCAB_FILES_NAMES
							__a   :				str       =    PRETRAINED_VOCAB_FILES_MAP
							__a   :				int       =    PRETRAINED_INIT_CONFIGURATION
							__a   :				int       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a   :				Tuple       =    LayoutLMTokenizer

							def __init__(							self							:       Dict    ,       __magic_name__							:       Tuple=None    ,       __magic_name__							:       Dict=None    ,       __magic_name__							:       str=True    ,       __magic_name__							:       Any="[UNK]"    ,       __magic_name__							:       Optional[int]="[SEP]"    ,       __magic_name__							:       Tuple="[PAD]"    ,       __magic_name__							:       Dict="[CLS]"    ,       __magic_name__							:       str="[MASK]"    ,       __magic_name__							:       Dict=True    ,       __magic_name__							:       Optional[int]=None    ,       **__magic_name__							:       Optional[Any]    ,       )   ->    Optional[Any]:







													"""simple docstring"""
													super().__init__(
													    __magic_name__    ,       tokenizer_file=__magic_name__    ,       do_lower_case=__magic_name__    ,       unk_token=__magic_name__    ,       sep_token=__magic_name__    ,       pad_token=__magic_name__    ,       cls_token=__magic_name__    ,       mask_token=__magic_name__    ,       tokenize_chinese_chars=__magic_name__    ,       strip_accents=__magic_name__    ,       **__magic_name__    ,       )

													UpperCAmelCase_     : Optional[Any]         =					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
													if (
													    normalizer_state.get('''lowercase'''    ,       __magic_name__				) != do_lower_case
													    or normalizer_state.get('''strip_accents'''    ,       __magic_name__				) != strip_accents
													    or normalizer_state.get('''handle_chinese_chars'''    ,       __magic_name__				) != tokenize_chinese_chars
													):
																			UpperCAmelCase_     : str         =					getattr(__magic_name__    ,       normalizer_state.pop('''type'''				)				)
																			UpperCAmelCase_     : Tuple         =					do_lower_case
																			UpperCAmelCase_     : str         =					strip_accents
																			UpperCAmelCase_     : int         =					tokenize_chinese_chars
																			UpperCAmelCase_     : Any         =					normalizer_class(**__magic_name__				)

													UpperCAmelCase_     : List[str]         =					do_lower_case

							def       UpperCAmelCase__		(							self							:       Optional[Any]    ,       __magic_name__							:       List[Any]    ,       __magic_name__							:       List[str]=None				)   ->    Any:







													"""simple docstring"""
													UpperCAmelCase_     : Any         =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																			output += token_ids_a + [self.sep_token_id]

													return output

							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       List[int]    ,       __magic_name__							:       Optional[List[int]] = None				)   ->    List[int]:







													"""simple docstring"""
													UpperCAmelCase_     : Optional[int]         =					[self.sep_token_id]
													UpperCAmelCase_     : Union[str, Any]         =					[self.cls_token_id]
													if token_ids_a is None:
																			return len(cls + token_ids_a + sep				) * [0]
													return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



							def       UpperCAmelCase__		(							self							:       int    ,       __magic_name__							:       str    ,       __magic_name__							:       Optional[str] = None				)   ->    Tuple[str]:







													"""simple docstring"""
													UpperCAmelCase_     : str         =					self._tokenizer.model.save(__magic_name__    ,       name=__magic_name__				)
													return tuple(__magic_name__				)












'''simple docstring'''







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


snake_case_       :    List[Any]          =					{
    "configuration_tvlt": ["TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP", "TvltConfig"],
    "feature_extraction_tvlt": ["TvltFeatureExtractor"],
    "processing_tvlt": ["TvltProcessor"],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_       :    Optional[int]          =					[
					    "TVLT_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TvltModel",
					    "TvltForPreTraining",
					    "TvltForAudioVisualClassification",
					    "TvltPreTrainedModel",
					]

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_       :    List[Any]          =					["TvltImageProcessor"]


if TYPE_CHECKING:
					from .configuration_tvlt import TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP, TvltConfig
					from .processing_tvlt import TvltProcessor
					from .feature_extraction_tvlt import TvltFeatureExtractor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tvlt import (
										    TVLT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TvltForAudioVisualClassification,
										    TvltForPreTraining,
										    TvltModel,
										    TvltPreTrainedModel,
										)

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .image_processing_tvlt import TvltImageProcessor


else:
					import sys

					snake_case_       :    Any          =					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





