






"""simple docstring"""



import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil
@pytest.fixture
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  str   ):





  '''simple docstring'''





  lowercase							   =	tmp_path / 'text.txt'
  lowercase							   =	textwrap.dedent(
      '\\n        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n\n        Second paragraph:\n        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n        '   )
  with open(__snake_case		, 'w'		, encoding='utf-8'   ) as f:
    f.write(__snake_case   )
  return str(__snake_case   )
@pytest.fixture
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  str		, __snake_case      :  Optional[Any]   ):





  '''simple docstring'''





  lowercase							   =	tmp_path / 'text_with_image.txt'
  with open(__snake_case		, 'w'		, encoding='utf-8'   ) as f:
    f.write(__snake_case   )
  return str(__snake_case   )
@pytest.mark.parametrize('keep_linebreaks'		, [True, False]   )
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  Union[str, Any]		, __snake_case      :  Optional[Any]   ):





  '''simple docstring'''





  with open(__snake_case		, encoding='utf-8'   ) as f:
    lowercase							   =	f.read().splitlines(keepends=__snake_case   )
  lowercase							   =	Text(keep_linebreaks=__snake_case		, encoding='utf-8'   )
  lowercase							   =	text._generate_tables([[text_file]]   )
  lowercase							   =	pa.concat_tables([table for _, table in generator]   ).to_pydict()['text']
  assert generated_content == expected_content
@require_pil
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  int   ):





  '''simple docstring'''





  with open(__snake_case		, encoding='utf-8'   ) as f:
    lowercase							   =	f.read().splitlines()[0]
  lowercase							   =	Text(encoding='utf-8'		, features=Features({'image': Image()}   )   )
  lowercase							   =	text._generate_tables([[text_file_with_image]]   )
  lowercase							   =	pa.concat_tables([table for _, table in generator]   )
  assert pa_table.schema.field('image'   ).type == Image()()
  lowercase							   =	pa_table.to_pydict()['image']
  assert generated_content == [{"path": image_file, "bytes": None}]



@pytest.mark.parametrize('sample_by'		, ['line', 'paragraph', 'document']   )
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  Union[str, Any]		, __snake_case      :  Optional[int]   ):





  '''simple docstring'''





  with open(__snake_case		, encoding='utf-8'   ) as f:
    lowercase							   =	f.read()
  if sample_by == "line":
    lowercase							   =	expected_content.splitlines()
  elif sample_by == "paragraph":
    lowercase							   =	expected_content.split('\n\n'   )
  elif sample_by == "document":
    lowercase							   =	[expected_content]
  lowercase							   =	Text(sample_by=__snake_case		, encoding='utf-8'		, chunksize=1_00   )
  lowercase							   =	text._generate_tables([[text_file]]   )
  lowercase							   =	pa.concat_tables([table for _, table in generator]   ).to_pydict()['text']
  assert generated_content == expected_content










"""simple docstring"""



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        RemBertConfig,
        RemBertForCausalLM,
        RemBertForMaskedLM,
        RemBertForMultipleChoice,
        RemBertForQuestionAnswering,
        RemBertForSequenceClassification,
        RemBertForTokenClassification,
        RemBertModel,
    )
    from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class 	a					:







    def __init__(      self							,		_lowerCamelCase							,		_lowerCamelCase=1_3							,		_lowerCamelCase=7							,		_lowerCamelCase=True							,		_lowerCamelCase=True							,		_lowerCamelCase=True							,		_lowerCamelCase=True							,		_lowerCamelCase=9_9							,		_lowerCamelCase=3_2							,		_lowerCamelCase=1_8							,		_lowerCamelCase=4_3							,		_lowerCamelCase=5							,		_lowerCamelCase=4							,		_lowerCamelCase=3_7							,		_lowerCamelCase="gelu"							,		_lowerCamelCase=0.1							,		_lowerCamelCase=0.1							,		_lowerCamelCase=5_1_2							,		_lowerCamelCase=1_6							,		_lowerCamelCase=2							,		_lowerCamelCase=0.0_2							,		_lowerCamelCase=3							,		_lowerCamelCase=4							,		_lowerCamelCase=None							,		):
      lowercase							   =	parent
      lowercase							   =	batch_size
      lowercase							   =	seq_length
      lowercase							   =	is_training
      lowercase							   =	use_input_mask
      lowercase							   =	use_token_type_ids
      lowercase							   =	use_labels
      lowercase							   =	vocab_size
      lowercase							   =	hidden_size
      lowercase							   =	input_embedding_size
      lowercase							   =	output_embedding_size
      lowercase							   =	num_hidden_layers
      lowercase							   =	num_attention_heads
      lowercase							   =	intermediate_size
      lowercase							   =	hidden_act
      lowercase							   =	hidden_dropout_prob
      lowercase							   =	attention_probs_dropout_prob
      lowercase							   =	max_position_embeddings
      lowercase							   =	type_vocab_size
      lowercase							   =	type_sequence_label_size
      lowercase							   =	initializer_range
      lowercase							   =	num_labels
      lowercase							   =	num_choices
      lowercase							   =	scope







    def       UpperCamelCase_				(      self	):
      lowercase							   =	ids_tensor([self.batch_size, self.seq_length]							,		self.vocab_size	)

      lowercase							   =	None
      if self.use_input_mask:
        lowercase							   =	random_attention_mask([self.batch_size, self.seq_length]	)

      lowercase							   =	None
      if self.use_token_type_ids:
        lowercase							   =	ids_tensor([self.batch_size, self.seq_length]							,		self.type_vocab_size	)

      lowercase							   =	None
      lowercase							   =	None
      lowercase							   =	None
      if self.use_labels:
        lowercase							   =	ids_tensor([self.batch_size]							,		self.type_sequence_label_size	)
        lowercase							   =	ids_tensor([self.batch_size, self.seq_length]							,		self.num_labels	)
        lowercase							   =	ids_tensor([self.batch_size]							,		self.num_choices	)

      lowercase							   =	RemBertConfig(
          vocab_size=self.vocab_size							,		hidden_size=self.hidden_size							,		input_embedding_size=self.input_embedding_size							,		output_embedding_size=self.output_embedding_size							,		num_hidden_layers=self.num_hidden_layers							,		num_attention_heads=self.num_attention_heads							,		intermediate_size=self.intermediate_size							,		hidden_act=self.hidden_act							,		hidden_dropout_prob=self.hidden_dropout_prob							,		attention_probs_dropout_prob=self.attention_probs_dropout_prob							,		max_position_embeddings=self.max_position_embeddings							,		type_vocab_size=self.type_vocab_size							,		is_decoder=_lowerCamelCase							,		initializer_range=self.initializer_range							,		)

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







    def       UpperCamelCase_				(      self	):
      (
          (
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					
      )							   =	self.prepare_config_and_inputs()

      lowercase							   =	True
      lowercase							   =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
      lowercase							   =	ids_tensor([self.batch_size, self.seq_length]							,		vocab_size=2	)

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	RemBertModel(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase	)
      lowercase							   =	model(_lowerCamelCase							,		token_type_ids=_lowerCamelCase	)
      lowercase							   =	model(_lowerCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape							,		(self.batch_size, self.seq_length, self.hidden_size)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		):
      lowercase							   =	True
      lowercase							   =	RemBertModel(_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		encoder_hidden_states=_lowerCamelCase							,		encoder_attention_mask=_lowerCamelCase							,		)
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		encoder_hidden_states=_lowerCamelCase							,		)
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape							,		(self.batch_size, self.seq_length, self.hidden_size)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		):
      lowercase							   =	RemBertForCausalLM(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		labels=_lowerCamelCase	)
      self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.vocab_size)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	RemBertForMaskedLM(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		labels=_lowerCamelCase	)
      self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.vocab_size)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		):
      lowercase							   =	True
      lowercase							   =	True
      lowercase							   =	RemBertForCausalLM(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()

      # first forward pass
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		encoder_hidden_states=_lowerCamelCase							,		encoder_attention_mask=_lowerCamelCase							,		use_cache=_lowerCamelCase							,		)
      lowercase							   =	outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      lowercase							   =	ids_tensor((self.batch_size, 3)							,		config.vocab_size	)
      lowercase							   =	ids_tensor((self.batch_size, 3)							,		vocab_size=2	)

      # append to next input_ids and
      lowercase							   =	torch.cat([input_ids, next_tokens]							,		dim=-1	)
      lowercase							   =	torch.cat([input_mask, next_mask]							,		dim=-1	)

      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		encoder_hidden_states=_lowerCamelCase							,		encoder_attention_mask=_lowerCamelCase							,		output_hidden_states=_lowerCamelCase							,		)['hidden_states'][0]
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		encoder_hidden_states=_lowerCamelCase							,		encoder_attention_mask=_lowerCamelCase							,		past_key_values=_lowerCamelCase							,		output_hidden_states=_lowerCamelCase							,		)['hidden_states'][0]

      # select random slice
      lowercase							   =	ids_tensor((1,)							,		output_from_past.shape[-1]	).item()
      lowercase							   =	output_from_no_past[:, -3:, random_slice_idx].detach()
      lowercase							   =	output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(_lowerCamelCase							,		_lowerCamelCase							,		atol=1e-3	)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	RemBertForQuestionAnswering(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		start_positions=_lowerCamelCase							,		end_positions=_lowerCamelCase							,		)
      self.parent.assertEqual(result.start_logits.shape							,		(self.batch_size, self.seq_length)	)
      self.parent.assertEqual(result.end_logits.shape							,		(self.batch_size, self.seq_length)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	self.num_labels
      lowercase							   =	RemBertForSequenceClassification(_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		labels=_lowerCamelCase	)
      self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.num_labels)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	self.num_labels
      lowercase							   =	RemBertForTokenClassification(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	model(_lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		labels=_lowerCamelCase	)
      self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.num_labels)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase	):
      lowercase							   =	self.num_choices
      lowercase							   =	RemBertForMultipleChoice(config=_lowerCamelCase	)
      model.to(_lowerCamelCase	)
      model.eval()
      lowercase							   =	input_ids.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
      lowercase							   =	token_type_ids.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
      lowercase							   =	input_mask.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
      lowercase							   =	model(
          _lowerCamelCase							,		attention_mask=_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		labels=_lowerCamelCase							,		)
      self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.num_choices)	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.prepare_config_and_inputs()
      (
          (
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					
      )							   =	config_and_inputs
      lowercase							   =	{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
      return config, inputs_dict


@require_torch
class 	a					(       a_,    a_,    unittest.TestCase     ):
    UpperCAmelCase_			:					int     					=(
        (
            RemBertModel,
            RemBertForMaskedLM,
            RemBertForCausalLM,
            RemBertForMultipleChoice,
            RemBertForQuestionAnswering,
            RemBertForSequenceClassification,
            RemBertForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    UpperCAmelCase_			:					Optional[int]     					=(RemBertForCausalLM,) if is_torch_available() else ()
    UpperCAmelCase_			:					Dict     					=(
        {
            "feature-extraction": RemBertModel,
            "fill-mask": RemBertForMaskedLM,
            "question-answering": RemBertForQuestionAnswering,
            "text-classification": RemBertForSequenceClassification,
            "text-generation": RemBertForCausalLM,
            "token-classification": RemBertForTokenClassification,
            "zero-shot": RemBertForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )







    def       UpperCamelCase_				(      self	):
      lowercase							   =	RemBertModelTester(self	)
      lowercase							   =	ConfigTester(self							,		config_class=_lowerCamelCase							,		hidden_size=3_7	)







    def       UpperCamelCase_				(      self	):
      self.config_tester.run_common_tests()







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
        lowercase							   =	type
        self.model_tester.create_and_check_model(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_decoder_model_past_large_inputs(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      lowercase							   =	self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_model_as_decoder(*_lowerCamelCase	)







    def       UpperCamelCase_				(      self	):
      # This regression test was failing with PyTorch < 1.3
      (
          (
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					(
          lowercase
      )   ,					
      )							   =	self.model_tester.prepare_config_and_inputs_for_decoder()

      lowercase							   =	None

      self.model_tester.create_and_check_model_as_decoder(
          _lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase							,		)







    @slow
    def       UpperCamelCase_				(      self	):
      for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase							   =	RemBertModel.from_pretrained(_lowerCamelCase	)
        self.assertIsNotNone(_lowerCamelCase	)


@require_torch
class 	a					(       unittest.TestCase     ):







    @slow
    def       UpperCamelCase_				(      self	):
      # Test exact values at the last hidden layer
      lowercase							   =	RemBertModel.from_pretrained('google/rembert'	)
      lowercase							   =	torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]	)
      lowercase							   =	torch.tensor([[0, 0, 0, 1, 1]]	)
      with torch.no_grad():
        lowercase							   =	model(_lowerCamelCase							,		token_type_ids=_lowerCamelCase							,		output_hidden_states=_lowerCamelCase	)

      lowercase							   =	1_1_5_2

      lowercase							   =	torch.Size((1, 5, hidden_size)	)
      self.assertEqual(output['last_hidden_state'].shape							,		_lowerCamelCase	)

      lowercase							   =	torch.tensor(
          [
              [
                  [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
                  [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
                  [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
                  [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
                  [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
              ]
          ]	)

      # Running on the original tf implementation gives slightly different results here.
      # Not clear why this variations is present
      # TODO: Find reason for discrepancy
      # expected_original_implementation = [[
      #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
      #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
      #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
      #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
      #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
      # ]]

      self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]							,		_lowerCamelCase							,		atol=1e-4	)	)


