



import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
				import torch

				from transformers import GPTaLMHeadModel

@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase__      (      unittest.TestCase       ):




			'''simple docstring'''






			@slow
			@unittest.skip('''Model is not available.'''			)
			def 			snake_case__					(       self							:			Optional[Any]			):
					'''simple docstring'''





					__UpperCAmelCase   :					int													=     '''nvidia/megatron-gpt2-345m/'''
					if "MYDIR" in os.environ:
							__UpperCAmelCase   :					List[str]													=     os.path.join(os.environ['''MYDIR''']	,	a_			)
					__UpperCAmelCase   :					List[str]													=     GPTaLMHeadModel.from_pretrained(a_			)
					model.to(a_			)
					model.half()

					__UpperCAmelCase   :					Tuple													=     torch.tensor(
					    [[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]	,	device=a_	,	dtype=torch.long	,	)

					with torch.no_grad():
							__UpperCAmelCase   :					Tuple													=     model(a_			).logits

					__UpperCAmelCase   :					Optional[int]													=     torch.Size((1, 9, 5_02_57)			)
					self.assertEqual(output.shape	,	a_			)

					__UpperCAmelCase   :					str													=     torch.tensor(
					    [
					        4.9_4_1_4,
					        -0.2_9_2_0,
					        -1.2_1_4_8,
					        -4.0_2_7_3,
					        -0.5_1_6_1,
					        -5.2_1_0_9,
					        -1.2_4_1_2,
					        -1.8_3_0_1,
					        -1.7_7_3_4,
					        -4.7_1_4_8,
					        -0.2_3_1_7,
					        -1.0_8_1_1,
					        -2.1_7_7_7,
					        0.4_1_4_1,
					        -3.7_9_6_9,
					        -4.0_5_8_6,
					        -2.5_3_3_2,
					        -3.3_8_0_9,
					        4.3_8_6_7,
					    ]	,	device=a_	,	dtype=torch.half	,	)

					for i in range(19			):
							__UpperCAmelCase							, __UpperCAmelCase   :					Any													=     8 * i // 17, 27_92 * i  # along the diagonal
							__UpperCAmelCase							, __UpperCAmelCase   :					Optional[int]													=     output[0, r, c], expected_diag[i]
							__UpperCAmelCase   :					Dict													=     F'row={r} col={c} computed={computed} expected={expected}'
							self.assertAlmostEqual(a_	,	a_	,	delta=1e-4	,	msg=a_			)










import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir

@dataclass
class 		UpperCAmelCase__      :




			'''simple docstring'''






			UpperCamelCase      						=	42

class 		UpperCAmelCase__      (      unittest.TestCase       ):




			'''simple docstring'''






			def 			snake_case__					(       self							:			Any			):
					'''simple docstring'''





					__UpperCAmelCase   :					Optional[int]													=     get_tests_dir('''fixtures/test_sentencepiece.model'''			)
					__UpperCAmelCase   :					Union[str, Any]													=     get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model'''			)

					__UpperCAmelCase   :					List[str]													=     FakeOriginalTokenizer(vocab_file=a_			)

					with warnings.catch_warnings(record=a_			) as w:
							__UpperCAmelCase   :					Optional[int]													=     SpmConverter(a_			)
					self.assertEqual(len(a_			)	,	0			)

					__UpperCAmelCase   :					Optional[int]													=     FakeOriginalTokenizer(vocab_file=a_			)

					with warnings.catch_warnings(record=a_			) as w:
							__UpperCAmelCase   :					Dict													=     SpmConverter(a_			)
					self.assertEqual(len(a_			)	,	1			)

					self.assertIn(
					    '''The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'''
					    ''' which is not implemented in the fast tokenizers.'''	,	str(w[0].message			)	,	)





