




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


SCREAMING_SNAKE_CASE__	:				int  =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__	:				List[str]  =						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE__	:				List[str]  =						{
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE__	:				Optional[int]  =						{
    'microsoft/layoutlm-base-uncased': 512,
    'microsoft/layoutlm-large-uncased': 512,
}

SCREAMING_SNAKE_CASE__	:				Union[str, Any]  =						{
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}




class 		UpperCamelCase__     (lowerCamelCase_					):

    '''simple docstring'''



    lowerCamelCase_				:     List[Any]     					=				VOCAB_FILES_NAMES
    lowerCamelCase_				:     int     					=				PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_				:     Any     					=				PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase_				:     str     					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_				:     int     					=				LayoutLMTokenizer







    def __init__(   self					,  UpperCamelCase__=None					,  UpperCamelCase__=None					,  UpperCamelCase__=True					,  UpperCamelCase__="[UNK]"					,  UpperCamelCase__="[SEP]"					,  UpperCamelCase__="[PAD]"					,  UpperCamelCase__="[CLS]"					,  UpperCamelCase__="[MASK]"					,  UpperCamelCase__=True					,  UpperCamelCase__=None					,  **UpperCamelCase__					,  )					->				str:
           super().__init__(
               UpperCamelCase__					,  tokenizer_file=UpperCamelCase__					,  do_lower_case=UpperCamelCase__					,  unk_token=UpperCamelCase__					,  sep_token=UpperCamelCase__					,  pad_token=UpperCamelCase__					,  cls_token=UpperCamelCase__					,  mask_token=UpperCamelCase__					,  tokenize_chinese_chars=UpperCamelCase__					,  strip_accents=UpperCamelCase__					,  **UpperCamelCase__					,  )

           lowerCamelCase       :				Tuple      			= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
           if (
               normalizer_state.get("lowercase"					,  UpperCamelCase__					) != do_lower_case
               or normalizer_state.get("strip_accents"					,  UpperCamelCase__					) != strip_accents
               or normalizer_state.get("handle_chinese_chars"					,  UpperCamelCase__					) != tokenize_chinese_chars
           ):
                  lowerCamelCase       :				str      			= getattr(UpperCamelCase__					,  normalizer_state.pop("type"					)					)
                  lowerCamelCase       :				Tuple      			= do_lower_case
                  lowerCamelCase       :				List[str]      			= strip_accents
                  lowerCamelCase       :				str      			= tokenize_chinese_chars
                  lowerCamelCase       :				Tuple      			= normalizer_class(**UpperCamelCase__					)

           lowerCamelCase       :				Tuple      			= do_lower_case







    def    _lowercase					(   self					,  UpperCamelCase__					,  UpperCamelCase__=None					)					->				Optional[Any]:
           lowerCamelCase       :				Optional[Any]      			= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def    _lowercase					(   self					,  UpperCamelCase__					,  UpperCamelCase__ = None					)					->				Any:
           lowerCamelCase       :				Optional[Any]      			= [self.sep_token_id]
           lowerCamelCase       :				Any      			= [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep					) * [0]
           return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







    def    _lowercase					(   self					,  UpperCamelCase__					,  UpperCamelCase__ = None					)					->				List[str]:
           lowerCamelCase       :				List[Any]      			= self._tokenizer.model.save(UpperCamelCase__					,  name=UpperCamelCase__					)
           return tuple(UpperCamelCase__					)












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__	:				Optional[int]  =       {
    'configuration_gptsan_japanese': ['GPTSAN_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTSanJapaneseConfig'],
    'tokenization_gptsan_japanese': ['GPTSanJapaneseTokenizer'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__	:				Optional[int]  =       [
           'GPTSAN_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST',
           'GPTSanJapaneseForConditionalGeneration',
           'GPTSanJapaneseModel',
           'GPTSanJapanesePreTrainedModel',
       ]
       SCREAMING_SNAKE_CASE__	:				List[str]  =       [
           'GPTSanJapaneseTokenizer',
       ]


if TYPE_CHECKING:
       from .configuration_gptsan_japanese import GPTSAN_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTSanJapaneseConfig
       from .tokenization_gptsan_japanese import GPTSanJapaneseTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_gptsan_japanese import (
                  GPTSAN_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
                  GPTSanJapaneseForConditionalGeneration,
                  GPTSanJapaneseModel,
                  GPTSanJapanesePreTrainedModel,
              )
              from .tokenization_gptsan_japanese import GPTSanJapaneseTokenizer


else:
       import sys

       SCREAMING_SNAKE_CASE__	:				Optional[Any]  =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






