






"""simple docstring"""





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig






def     A(  snake_case_						,       snake_case_						,       snake_case_=0		):
       """simple docstring"""





       if name is None:
              lowercase__:    Tuple        =      None
       else:
              lowercase__:    List[Any]        =      "." * max(0						,       spaces - 2		) + "# {:" + str(50 - spaces		) + "s}"
              lowercase__:    Tuple        =      fmt.format(snake_case_		)

       # Print and recurse (if needed).
       if isinstance(snake_case_						,       snake_case_		):
              if msg is not None:
                     print(snake_case_		)
              for k in val.keys():
                     recursive_print(snake_case_						,       val[k]						,       spaces + 2		)
       elif isinstance(snake_case_						,       torch.Tensor		):
              print(snake_case_						,       ":"						,       val.size()		)
       else:
              print(snake_case_						,       ":"						,       snake_case_		)






def     A(  snake_case_						,       snake_case_						,       snake_case_						,       snake_case_						,       snake_case_		):
       """simple docstring"""





       lowercase__:    Union[str, Any]        =      param.size()
       if checkpoint_version == 1.0:
              # version 1.0 stores [num_heads * hidden_size * num_splits, :]
              lowercase__:    str        =      (num_heads, hidden_size, num_splits) + input_shape[1:]
              lowercase__:    Optional[Any]        =      param.view(*snake_case_		)
              lowercase__:    Optional[int]        =      param.transpose(0						,       2		)
              lowercase__:    Optional[Any]        =      param.transpose(1						,       2		).contiguous()
       elif checkpoint_version >= 2.0:
              # other versions store [num_heads * num_splits * hidden_size, :]
              lowercase__:    List[Any]        =      (num_heads, num_splits, hidden_size) + input_shape[1:]
              lowercase__:    Optional[int]        =      param.view(*snake_case_		)
              lowercase__:    List[Any]        =      param.transpose(0						,       1		).contiguous()
       lowercase__:    Any        =      param.view(*snake_case_		)
       return param






def     A(  snake_case_						,       snake_case_						,       snake_case_		):
       """simple docstring"""





       lowercase__:    Any        =      {}

       # old versions did not store training args
       lowercase__:    int        =      input_state_dict.get("args"						,       snake_case_		)
       if ds_args is not None:
              # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
              # from pprint import pprint
              # pprint(vars(ds_args))

              lowercase__:    Optional[int]        =      ds_args.tokenizer_type
              lowercase__:    List[Any]        =      ds_args.padded_vocab_size
              lowercase__:    Tuple        =      ds_args.max_position_embeddings
              lowercase__:    Any        =      ds_args.hidden_size
              lowercase__:    str        =      ds_args.num_layers
              lowercase__:    Dict        =      ds_args.num_attention_heads
              lowercase__:    str        =      ds_args.ffn_hidden_size if "ffn_hidden_size" in ds_args else 4 * ds_args.hidden_size
              # pprint(config)

       # The number of heads.
       lowercase__:    List[Any]        =      config.num_attention_heads
       # The hidden_size per head.
       lowercase__:    Optional[int]        =      config.hidden_size // heads
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
              lowercase__:    Dict        =      input_state_dict["checkpoint_version"]
       else:
              lowercase__:    str        =      0.0

       # The model.
       lowercase__:    str        =      input_state_dict["model"]
       # The language model.
       lowercase__:    str        =      model["language_model"]
       # The embeddings.
       lowercase__:    str        =      lm["embedding"]

       # The word embeddings.
       lowercase__:    str        =      embeddings["word_embeddings"]["weight"]
       # Truncate the embedding table to vocab_size rows.
       lowercase__:    int        =      word_embeddings[: config.vocab_size, :]
       # Store the word embeddings.
       lowercase__:    List[Any]        =      word_embeddings

       # The position embeddings.
       lowercase__:    Any        =      embeddings["position_embeddings"]["weight"]
       assert pos_embeddings.size(0		) == config.max_position_embeddings and pos_embeddings.size(1		) == config.hidden_size
       # Store the position embeddings.
       lowercase__:    Union[str, Any]        =      pos_embeddings

       # The token-type embeddings.
       lowercase__:    Union[str, Any]        =      embeddings["tokentype_embeddings"]["weight"]
       # Store the position embeddings.
       lowercase__:    Optional[Any]        =      tokentype_embeddings

       # The transformer.
       lowercase__:    List[Any]        =      lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

       # The regex to extract layer names.
       lowercase__:    Dict        =      re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"		)

       # The simple map of names for "automated" rules.
       lowercase__:    Any        =      {
           "attention.dense": ".attention.output.dense.",
           "self_attention.dense": ".attention.output.dense.",
           "mlp.dense_h_to_4h": ".intermediate.dense.",
           "mlp.dense_4h_to_h": ".output.dense.",
       }

       # Keep track of the attention/query/value tensor.
       lowercase__:    Union[str, Any]        =      None

       # Extract the layers.
       for key, val in transformer.items():
              # Match the name.
              lowercase__:    List[Any]        =      layer_re.match(snake_case_		)

              # Stop if that's not a layer
              if m is None:
                     break

              # The index of the layer.
              lowercase__:    Dict        =      int(m.group(1		)		)
              # The name of the operation.
              lowercase__:    List[str]        =      m.group(2		)
              # Is it a weight or a bias?
              lowercase__:    Optional[Any]        =      m.group(3		)

              # The name of the layer.
              lowercase__:    int        =      F"""bert.encoder.layer.{layer_idx}"""

              # For layernorm(s), simply store the layer norm.
              if op_name.endswith("layernorm"		):
                     lowercase__:    Optional[Any]        =      "attention.ln" if op_name.startswith("input"		) else "ln"
                     lowercase__:    Optional[Any]        =      val

              # Transpose the QKV matrix.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "weight":
                     # Make sure the QKV pointer is nil.
                     assert attention_qkv_weight is None, ""

                     lowercase__:    int        =      fix_query_key_value_ordering(snake_case_						,       snake_case_						,       3						,       snake_case_						,       snake_case_		)
                     # Store the tensor as we need the bias as well to interleave QKV and biases.
                     lowercase__:    Dict        =      out_val

              # Transpose the bias.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "bias":
                     # Make sure we read the weight tensor.
                     assert attention_qkv_weight is not None, ""

                     # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
                     lowercase__:    Dict        =      attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
                     lowercase__:    Optional[Any]        =      attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
                     lowercase__:    Optional[Any]        =      attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

                     lowercase__:    Dict        =      fix_query_key_value_ordering(snake_case_						,       snake_case_						,       3						,       snake_case_						,       snake_case_		)
                     # Split the bias.
                     lowercase__:    Dict        =      out_val[0 * config.hidden_size : 1 * config.hidden_size]
                     lowercase__:    Optional[int]        =      out_val[1 * config.hidden_size : 2 * config.hidden_size]
                     lowercase__:    int        =      out_val[2 * config.hidden_size : 3 * config.hidden_size]

                     # Store.
                     lowercase__:    Tuple        =      q
                     lowercase__:    Union[str, Any]        =      q_bias
                     lowercase__:    Tuple        =      k
                     lowercase__:    int        =      k_bias
                     lowercase__:    int        =      v
                     lowercase__:    Tuple        =      v_bias

                     # Clear the stored tensor.
                     lowercase__:    Dict        =      None

              # Copy weights and biases as is.
              elif weight_or_bias in ["weight", "bias"]:
                     lowercase__:    Tuple        =      megatron_to_transformers[op_name]
                     lowercase__:    Optional[int]        =      val

    # The final layernorm.
       lowercase__:    Optional[Any]        =      transformer["final_layernorm.weight"]
       lowercase__:    List[str]        =      transformer["final_layernorm.bias"]

       # The pooler.
       lowercase__:    Optional[Any]        =      lm["pooler"]

       # Store the matrix and the bias.
       lowercase__:    Any        =      pooler["dense.weight"]
       lowercase__:    Optional[int]        =      pooler["dense.bias"]

       # The LM head from Megatron (for RACE).
       lowercase__:    str        =      model["lm_head"]

       # The transform matrix.
       lowercase__:    str        =      lm_head["dense.weight"]
       lowercase__:    Optional[Any]        =      lm_head["dense.bias"]

       # The transform LN.
       lowercase__:    int        =      lm_head["layernorm.weight"]
       lowercase__:    Dict        =      lm_head["layernorm.bias"]

       # For the decoder, we replicate the weights.
       lowercase__:    Optional[int]        =      word_embeddings
       lowercase__:    Dict        =      lm_head["bias"]

       # The classifier from Megatron (for MLNI).
       lowercase__:    Optional[Any]        =      model["binary_head"]

       # Store the classifier.
       lowercase__:    Any        =      binary_head["weight"]
       lowercase__:    str        =      binary_head["bias"]

       # It should be done!
       return output_state_dict






def     A(  ):
       """simple docstring"""





       lowercase__:    Optional[int]        =      argparse.ArgumentParser()
       parser.add_argument("--print-checkpoint-structure"						,       action="store_true"		)
       parser.add_argument("path_to_checkpoint"						,       type=snake_case_						,       help="Path to the ZIP file containing the checkpoint"		)
       parser.add_argument(
           "--config_file"						,       default=""						,       type=snake_case_						,       help="An optional config json file describing the pre-trained model."						,       )
       lowercase__:    List[Any]        =      parser.parse_args()

       # Extract the basename.
       lowercase__:    Optional[Any]        =      os.path.dirname(args.path_to_checkpoint		)

       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(F"""Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""""		)
       if args.path_to_checkpoint.endswith(".zip"		):
              with zipfile.ZipFile(args.path_to_checkpoint						,       "r"		) as checkpoint:
                     with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"		) as pytorch_dict:
                            lowercase__:    Dict        =      torch.load(snake_case_						,       map_location="cpu"		)
       else:
              lowercase__:    List[Any]        =      torch.load(args.path_to_checkpoint						,       map_location="cpu"		)

       if args.config_file == "":
              # Default config of megatron-bert 345m
              lowercase__:    Optional[Any]        =      MegatronBertConfig()

              # different megatron-bert-*-345m models have different vocab sizes, so override the default
              # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
              lowercase__:    Union[str, Any]        =      input_state_dict["model"]["lm_head"]["bias"].numel()
       else:
              lowercase__:    List[Any]        =      MegatronBertConfig.from_json_file(args.config_file		)

       # Convert.
       print("Converting"		)
       lowercase__:    Optional[Any]        =      convert_megatron_checkpoint(snake_case_						,       snake_case_						,       snake_case_		)

       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
              recursive_print(snake_case_						,       snake_case_		)

       # Store the config to file.
       print("Saving config"		)
       config.save_pretrained(snake_case_		)

       # Store the state_dict to file.
       lowercase__:    Optional[int]        =      os.path.join(snake_case_						,       "pytorch_model.bin"		)
       print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""		)
       torch.save(snake_case_						,       snake_case_		)


####################################################################################################

if __name__ == "__main__":
  main()

####################################################################################################














"""simple docstring"""





import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      LayoutLMForMaskedLM,
      LayoutLMForQuestionAnswering,
      LayoutLMForSequenceClassification,
      LayoutLMForTokenClassification,
      LayoutLMModel,
  )







class 		_a       :





   '''simple docstring'''






   def __init__(   self			,				UpperCAmelCase_			,				UpperCAmelCase_=13			,				UpperCAmelCase_=7			,				UpperCAmelCase_=True			,				UpperCAmelCase_=True			,				UpperCAmelCase_=True			,				UpperCAmelCase_=True			,				UpperCAmelCase_=99			,				UpperCAmelCase_=32			,				UpperCAmelCase_=5			,				UpperCAmelCase_=4			,				UpperCAmelCase_=37			,				UpperCAmelCase_="gelu"			,				UpperCAmelCase_=0.1			,				UpperCAmelCase_=0.1			,				UpperCAmelCase_=512			,				UpperCAmelCase_=16			,				UpperCAmelCase_=2			,				UpperCAmelCase_=0.02			,				UpperCAmelCase_=3			,				UpperCAmelCase_=4			,				UpperCAmelCase_=None			,				UpperCAmelCase_=1_000			,				)	-> Optional[int]:




          '''simple docstring'''




          lowercase__:    int        =      parent
          lowercase__:    str        =      batch_size
          lowercase__:    Tuple        =      seq_length
          lowercase__:    Any        =      is_training
          lowercase__:    Union[str, Any]        =      use_input_mask
          lowercase__:    str        =      use_token_type_ids
          lowercase__:    Optional[int]        =      use_labels
          lowercase__:    List[str]        =      vocab_size
          lowercase__:    Optional[Any]        =      hidden_size
          lowercase__:    Optional[Any]        =      num_hidden_layers
          lowercase__:    Optional[int]        =      num_attention_heads
          lowercase__:    List[str]        =      intermediate_size
          lowercase__:    Any        =      hidden_act
          lowercase__:    Any        =      hidden_dropout_prob
          lowercase__:    List[str]        =      attention_probs_dropout_prob
          lowercase__:    Tuple        =      max_position_embeddings
          lowercase__:    Any        =      type_vocab_size
          lowercase__:    List[str]        =      type_sequence_label_size
          lowercase__:    Dict        =      initializer_range
          lowercase__:    Dict        =      num_labels
          lowercase__:    List[str]        =      num_choices
          lowercase__:    int        =      scope
          lowercase__:    List[Any]        =      range_bbox





   def __lowercase    (   self)	-> Optional[int]:




          '''simple docstring'''




          lowercase__:    int        =      ids_tensor([self.batch_size, self.seq_length]			,				self.vocab_size)

          lowercase__:    Tuple        =      ids_tensor([self.batch_size, self.seq_length, 4]			,				self.range_bbox)
          # Ensure that bbox is legal
          for i in range(bbox.shape[0]):
                 for j in range(bbox.shape[1]):
                        if bbox[i, j, 3] < bbox[i, j, 1]:
                               lowercase__:    List[str]        =      bbox[i, j, 3]
                               lowercase__:    List[str]        =      bbox[i, j, 1]
                               lowercase__:    Union[str, Any]        =      t
                        if bbox[i, j, 2] < bbox[i, j, 0]:
                               lowercase__:    Optional[int]        =      bbox[i, j, 2]
                               lowercase__:    Tuple        =      bbox[i, j, 0]
                               lowercase__:    Optional[Any]        =      t

          lowercase__:    Dict        =      None
          if self.use_input_mask:
                 lowercase__:    str        =      ids_tensor([self.batch_size, self.seq_length]			,				vocab_size=2)

          lowercase__:    str        =      None
          if self.use_token_type_ids:
                 lowercase__:    List[Any]        =      ids_tensor([self.batch_size, self.seq_length]			,				self.type_vocab_size)

          lowercase__:    Tuple        =      None
          lowercase__:    Dict        =      None
          lowercase__:    Any        =      None
          if self.use_labels:
                 lowercase__:    List[Any]        =      ids_tensor([self.batch_size]			,				self.type_sequence_label_size)
                 lowercase__:    Any        =      ids_tensor([self.batch_size, self.seq_length]			,				self.num_labels)
                 lowercase__:    List[str]        =      ids_tensor([self.batch_size]			,				self.num_choices)

          lowercase__:    int        =      self.get_config()

          return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





   def __lowercase    (   self)	-> Optional[int]:




          '''simple docstring'''




          return LayoutLMConfig(
              vocab_size=self.vocab_size			,				hidden_size=self.hidden_size			,				num_hidden_layers=self.num_hidden_layers			,				num_attention_heads=self.num_attention_heads			,				intermediate_size=self.intermediate_size			,				hidden_act=self.hidden_act			,				hidden_dropout_prob=self.hidden_dropout_prob			,				attention_probs_dropout_prob=self.attention_probs_dropout_prob			,				max_position_embeddings=self.max_position_embeddings			,				type_vocab_size=self.type_vocab_size			,				initializer_range=self.initializer_range			,				)





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> Optional[Any]:




          '''simple docstring'''




          lowercase__:    Any        =      LayoutLMModel(config=UpperCAmelCase_)
          model.to(UpperCAmelCase_)
          model.eval()
          lowercase__:    Tuple        =      model(UpperCAmelCase_			,				UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_)
          lowercase__:    Dict        =      model(UpperCAmelCase_			,				UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_)
          lowercase__:    Any        =      model(UpperCAmelCase_			,				UpperCAmelCase_)
          self.parent.assertEqual(result.last_hidden_state.shape			,				(self.batch_size, self.seq_length, self.hidden_size))
          self.parent.assertEqual(result.pooler_output.shape			,				(self.batch_size, self.hidden_size))





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> List[Any]:




          '''simple docstring'''




          lowercase__:    int        =      LayoutLMForMaskedLM(config=UpperCAmelCase_)
          model.to(UpperCAmelCase_)
          model.eval()
          lowercase__:    str        =      model(UpperCAmelCase_			,				UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				labels=UpperCAmelCase_)
          self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.seq_length, self.vocab_size))





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> Optional[int]:




          '''simple docstring'''




          lowercase__:    Union[str, Any]        =      self.num_labels
          lowercase__:    Optional[Any]        =      LayoutLMForSequenceClassification(UpperCAmelCase_)
          model.to(UpperCAmelCase_)
          model.eval()
          lowercase__:    Optional[Any]        =      model(
              UpperCAmelCase_			,				UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				labels=UpperCAmelCase_)
          self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.num_labels))





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> List[str]:




          '''simple docstring'''




          lowercase__:    List[str]        =      self.num_labels
          lowercase__:    List[str]        =      LayoutLMForTokenClassification(config=UpperCAmelCase_)
          model.to(UpperCAmelCase_)
          model.eval()
          lowercase__:    List[Any]        =      model(UpperCAmelCase_			,				UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				labels=UpperCAmelCase_)
          self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.seq_length, self.num_labels))





   def __lowercase    (   self			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_			,				UpperCAmelCase_)	-> str:




          '''simple docstring'''




          lowercase__:    Dict        =      LayoutLMForQuestionAnswering(config=UpperCAmelCase_)
          model.to(UpperCAmelCase_)
          model.eval()
          lowercase__:    List[Any]        =      model(
              UpperCAmelCase_			,				bbox=UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				start_positions=UpperCAmelCase_			,				end_positions=UpperCAmelCase_			,				)
          self.parent.assertEqual(result.start_logits.shape			,				(self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape			,				(self.batch_size, self.seq_length))





   def __lowercase    (   self)	-> Tuple:




          '''simple docstring'''




          lowercase__:    int        =      self.prepare_config_and_inputs()
          (
              (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     (
              lowercase__
          )  ,     
          ):    int        =      config_and_inputs
          lowercase__:    Optional[int]        =      {
              "input_ids": input_ids,
              "bbox": bbox,
              "token_type_ids": token_type_ids,
              "attention_mask": input_mask,
          }
          return config, inputs_dict







@require_torch
class 		_a       (						lowercase_    ,					lowercase_    ,					unittest.TestCase   ):





   '''simple docstring'''






   UpperCamelCase__      					=    (
       (
           LayoutLMModel,
           LayoutLMForMaskedLM,
           LayoutLMForSequenceClassification,
           LayoutLMForTokenClassification,
           LayoutLMForQuestionAnswering,
       )
       if is_torch_available()
       else None
   )
   UpperCamelCase__      					=    (
       {
           """document-question-answering""": LayoutLMForQuestionAnswering,
           """feature-extraction""": LayoutLMModel,
           """fill-mask""": LayoutLMForMaskedLM,
           """text-classification""": LayoutLMForSequenceClassification,
           """token-classification""": LayoutLMForTokenClassification,
           """zero-shot""": LayoutLMForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   UpperCamelCase__      					=    True





   def __lowercase    (   self)	-> Optional[int]:




          '''simple docstring'''




          lowercase__:    List[str]        =      LayoutLMModelTester(self)
          lowercase__:    List[str]        =      ConfigTester(self			,				config_class=UpperCAmelCase_			,				hidden_size=37)





   def __lowercase    (   self)	-> Optional[Any]:




          '''simple docstring'''




          self.config_tester.run_common_tests()





   def __lowercase    (   self)	-> List[str]:




          '''simple docstring'''




          lowercase__:    int        =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*UpperCAmelCase_)





   def __lowercase    (   self)	-> List[str]:




          '''simple docstring'''




          lowercase__:    List[Any]        =      self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
                 lowercase__:    Union[str, Any]        =      type
                 self.model_tester.create_and_check_model(*UpperCAmelCase_)





   def __lowercase    (   self)	-> List[str]:




          '''simple docstring'''




          lowercase__:    Any        =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase_)





   def __lowercase    (   self)	-> Dict:




          '''simple docstring'''




          lowercase__:    List[Any]        =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase_)





   def __lowercase    (   self)	-> int:




          '''simple docstring'''




          lowercase__:    Dict        =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase_)





   def __lowercase    (   self)	-> str:




          '''simple docstring'''




          lowercase__:    Optional[Any]        =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase_)





   @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests.")
   def __lowercase    (   self)	-> List[Any]:




          '''simple docstring'''




          pass







def     A(  ):
       """simple docstring"""





       lowercase__:    Any        =      torch.tensor([[101,1019,1014,1016,1037,12849,4747,1004,14246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,11300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,19274,2772,6205,27814,16147,16147,4343,2047,10283,10969,14389,1012,2338,102]]						,       device=snake_case_		)  # noqa: E231
       lowercase__:    Dict        =      torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]						,       device=snake_case_		)  # noqa: E231
       lowercase__:    Optional[Any]        =      torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]						,       device=snake_case_		)  # noqa: E231
       lowercase__:    Dict        =      torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]						,       device=snake_case_		)  # noqa: E231
       # these are sequence labels (i.e. at the token level)
       lowercase__:    int        =      torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]						,       device=snake_case_		)  # noqa: E231
       # fmt: on

       return input_ids, attention_mask, bbox, token_type_ids, labels







@require_torch
class 		_a       (						unittest.TestCase   ):





   '''simple docstring'''






   @slow
   def __lowercase    (   self)	-> Any:




          '''simple docstring'''




          lowercase__:    str        =      LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased").to(UpperCAmelCase_)

          lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__:    Dict        =      prepare_layoutlm_batch_inputs()

          # forward pass
          lowercase__:    str        =      model(input_ids=UpperCAmelCase_			,				bbox=UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_)

          # test the sequence output on [0, :3, :3]
          lowercase__:    Optional[int]        =      torch.tensor(
              [[0.17_85, -0.19_47, -0.04_25], [-0.32_54, -0.28_07, 0.25_53], [-0.53_91, -0.33_22, 0.33_64]]			,				device=UpperCAmelCase_			,				)

          self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,				UpperCAmelCase_			,				atol=1E-3))

          # test the pooled output on [1, :3]
          lowercase__:    List[Any]        =      torch.tensor([-0.65_80, -0.02_14, 0.85_52]			,				device=UpperCAmelCase_)

          self.assertTrue(torch.allclose(outputs.pooler_output[1, :3]			,				UpperCAmelCase_			,				atol=1E-3))





   @slow
   def __lowercase    (   self)	-> Optional[Any]:




          '''simple docstring'''




          lowercase__:    int        =      LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased"			,				num_labels=2).to(
              UpperCAmelCase_)

          lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__:    List[Any]        =      prepare_layoutlm_batch_inputs()

          # forward pass
          lowercase__:    str        =      model(
              input_ids=UpperCAmelCase_			,				bbox=UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				labels=torch.tensor([1, 1]			,				device=UpperCAmelCase_)			,				)

          # test whether we get a loss as a scalar
          lowercase__:    Dict        =      outputs.loss
          lowercase__:    int        =      torch.Size([])
          self.assertEqual(loss.shape			,				UpperCAmelCase_)

          # test the shape of the logits
          lowercase__:    str        =      outputs.logits
          lowercase__:    Any        =      torch.Size((2, 2))
          self.assertEqual(logits.shape			,				UpperCAmelCase_)





   @slow
   def __lowercase    (   self)	-> Union[str, Any]:




          '''simple docstring'''




          lowercase__:    Tuple        =      LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased"			,				num_labels=13).to(
              UpperCAmelCase_)

          lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__:    List[Any]        =      prepare_layoutlm_batch_inputs()

          # forward pass
          lowercase__:    Any        =      model(
              input_ids=UpperCAmelCase_			,				bbox=UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_			,				labels=UpperCAmelCase_)

          # test the loss calculation to be around 2.65
          # expected_loss = torch.tensor(2.65, device=torch_device)

          # The loss is currently somewhat random and can vary between 0.1-0.3 atol.
          # self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

          # test the shape of the logits
          lowercase__:    int        =      outputs.logits
          lowercase__:    int        =      torch.Size((2, 25, 13))
          self.assertEqual(logits.shape			,				UpperCAmelCase_)





   @slow
   def __lowercase    (   self)	-> Tuple:




          '''simple docstring'''




          lowercase__:    Tuple        =      LayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased").to(UpperCAmelCase_)

          lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__  ,     lowercase__:    Tuple        =      prepare_layoutlm_batch_inputs()

          # forward pass
          lowercase__:    str        =      model(input_ids=UpperCAmelCase_			,				bbox=UpperCAmelCase_			,				attention_mask=UpperCAmelCase_			,				token_type_ids=UpperCAmelCase_)

          # test the shape of the logits
          lowercase__:    str        =      torch.Size((2, 25))
          self.assertEqual(outputs.start_logits.shape			,				UpperCAmelCase_)
          self.assertEqual(outputs.end_logits.shape			,				UpperCAmelCase_)






