


import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


UpperCAmelCase__            =							namedtuple('''ExampleNamedTuple''', '''a b c''')







class 				lowerCamelCase__       (	unittest.TestCase):





     def 					__A (self	)					->       int:
      _lowercase											=torch.randn(5		,					2	)
      _lowercase											=torch.device('''cuda'''	) if torch.cuda.is_available() else torch.device('''cpu'''	)

      _lowercase											=send_to_device(UpperCAmelCase		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta.cpu()		,					UpperCAmelCase	)	)

      _lowercase											=send_to_device((tensor, [tensor, tensor], 1)		,					UpperCAmelCase	)
      self.assertIsInstance(UpperCAmelCase		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta[0].cpu()		,					UpperCAmelCase	)	)
      self.assertIsInstance(resulta[1]		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta[1][0].cpu()		,					UpperCAmelCase	)	)
      self.assertTrue(torch.equal(resulta[1][1].cpu()		,					UpperCAmelCase	)	)
      self.assertEqual(resulta[2]		,					1	)

      _lowercase											=send_to_device({'''a''': tensor, '''b''': [tensor, tensor], '''c''': 1}		,					UpperCAmelCase	)
      self.assertIsInstance(UpperCAmelCase		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta['''a'''].cpu()		,					UpperCAmelCase	)	)
      self.assertIsInstance(resulta['''b''']		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta['''b'''][0].cpu()		,					UpperCAmelCase	)	)
      self.assertTrue(torch.equal(resulta['''b'''][1].cpu()		,					UpperCAmelCase	)	)
      self.assertEqual(resulta['''c''']		,					1	)

      _lowercase											=send_to_device(ExampleNamedTuple(a=UpperCAmelCase		,					b=[tensor, tensor]		,					c=1	)		,					UpperCAmelCase	)
      self.assertIsInstance(UpperCAmelCase		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta.a.cpu()		,					UpperCAmelCase	)	)
      self.assertIsInstance(resulta.b		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta.b[0].cpu()		,					UpperCAmelCase	)	)
      self.assertTrue(torch.equal(resulta.b[1].cpu()		,					UpperCAmelCase	)	)
      self.assertEqual(resulta.c		,					1	)

      _lowercase											=send_to_device(UserDict({'''a''': tensor, '''b''': [tensor, tensor], '''c''': 1}	)		,					UpperCAmelCase	)
      self.assertIsInstance(UpperCAmelCase		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta['''a'''].cpu()		,					UpperCAmelCase	)	)
      self.assertIsInstance(resulta['''b''']		,					UpperCAmelCase	)
      self.assertTrue(torch.equal(resulta['''b'''][0].cpu()		,					UpperCAmelCase	)	)
      self.assertTrue(torch.equal(resulta['''b'''][1].cpu()		,					UpperCAmelCase	)	)
      self.assertEqual(resulta['''c''']		,					1	)





     def 					__A (self	)					->       Optional[Any]:
      with self.assertRaises(UpperCAmelCase	) as cm:
       _lowercase											=recursively_apply(torch.tensor		,					(torch.tensor(1	), 1)		,					error_on_other_type=UpperCAmelCase	)
      self.assertEqual(
          str(cm.exception	)		,					'''Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.'''		,					)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=torch.tensor([1, 2, 3, 4, 5]	)
      self.assertEqual(listify(UpperCAmelCase	)		,					[1, 2, 3, 4, 5]	)

      _lowercase											=torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]	)
      self.assertEqual(listify(UpperCAmelCase	)		,					[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]	)

      _lowercase											=torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]	)
      self.assertEqual(
          listify(UpperCAmelCase	)		,					[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]	)





     def 					__A (self	)					->       int:
      with patch_environment(aa=1		,					BB=2	):
       self.assertEqual(os.environ.get('''AA'''	)		,					'''1'''	)
       self.assertEqual(os.environ.get('''BB'''	)		,					'''2'''	)

      self.assertNotIn('''AA'''		,					os.environ	)
      self.assertNotIn('''BB'''		,					os.environ	)





     def 					__A (self	)					->       Optional[int]:
      _lowercase											=RegressionModel()
      _lowercase											=model.forward
      _lowercase											=convert_outputs_to_fpaa(model.forward	)
      _lowercase											=extract_model_from_parallel(UpperCAmelCase		,					keep_fpaa_wrapper=UpperCAmelCase	)
      _lowercase											=pickle.dumps(UpperCAmelCase	)





     @require_cuda
     def 					__A (self	)					->       List[str]:
      _lowercase											=RegressionModel()
      _lowercase											=model.forward
      _lowercase											=torch.cuda.amp.autocast(dtype=torch.floataa	)(model.forward	)
      _lowercase											=convert_outputs_to_fpaa(model.forward	)
      _lowercase											=extract_model_from_parallel(UpperCAmelCase		,					keep_fpaa_wrapper=UpperCAmelCase	)
      _lowercase											=pickle.dumps(UpperCAmelCase	)





     @require_cuda
     @require_torch_min_version(version='''2.0'''	)
     def 					__A (self	)					->       Optional[int]:
      _lowercase											=RegressionModel()
      _lowercase											=model.forward
      _lowercase											=torch.cuda.amp.autocast(dtype=torch.floataa	)(model.forward	)
      _lowercase											=convert_outputs_to_fpaa(model.forward	)
      _lowercase											=torch.compile(model.forward		,					backend='''inductor'''	)
      _lowercase											=torch.randn(4		,					1_0	).cuda()
      _lowercase											=model(UpperCAmelCase	)





     def 					__A (self	)					->       List[Any]:
      _lowercase											=RegressionModel()
      # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
      _lowercase											=torch.nn.parallel.DataParallel(UpperCAmelCase	)
      _lowercase											=extract_model_from_parallel(UpperCAmelCase	)

      self.assertEqual(UpperCAmelCase		,					UpperCAmelCase	)





     @require_torch_min_version(version='''2.0'''	)
     def 					__A (self	)					->       str:
      _lowercase											=RegressionModel()
      _lowercase											=torch.compile(UpperCAmelCase	)

      # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
      _lowercase											=torch.nn.parallel.DataParallel(UpperCAmelCase	)
      _lowercase											=torch.compile(UpperCAmelCase	)
      _lowercase											=extract_model_from_parallel(UpperCAmelCase	)

      self.assertEqual(compiled_model._orig_mod		,					compiled_model_unwrapped._orig_mod	)





     def 					__A (self	)					->       List[Any]:
      self.assertEqual(find_device([1, '''a''', torch.tensor([1, 2, 3]	)]	)		,					torch.device('''cpu'''	)	)
      self.assertEqual(find_device({'''a''': 1, '''b''': torch.tensor([1, 2, 3]	)}	)		,					torch.device('''cpu'''	)	)
      self.assertIsNone(find_device([1, '''a''']	)	)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__            =							logging.get_logger(__name__)

UpperCAmelCase__            =							{
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}







class 				lowerCamelCase__       (	lowerCAmelCase):
     SCREAMING_SNAKE_CASE__				      =      '''qdqbert'''





     def __init__(self		,					UpperCAmelCase=3_0_5_2_2		,					UpperCAmelCase=7_6_8		,					UpperCAmelCase=1_2		,					UpperCAmelCase=1_2		,					UpperCAmelCase=3_0_7_2		,					UpperCAmelCase="gelu"		,					UpperCAmelCase=0.1		,					UpperCAmelCase=0.1		,					UpperCAmelCase=5_1_2		,					UpperCAmelCase=2		,					UpperCAmelCase=0.02		,					UpperCAmelCase=1e-12		,					UpperCAmelCase=True		,					UpperCAmelCase=1		,					UpperCAmelCase=0		,					UpperCAmelCase=2		,					**UpperCAmelCase		,					)					->       str:
      super().__init__(pad_token_id=UpperCAmelCase		,					bos_token_id=UpperCAmelCase		,					eos_token_id=UpperCAmelCase		,					**UpperCAmelCase	)

      _lowercase											=vocab_size
      _lowercase											=max_position_embeddings
      _lowercase											=hidden_size
      _lowercase											=num_hidden_layers
      _lowercase											=num_attention_heads
      _lowercase											=intermediate_size
      _lowercase											=hidden_act
      _lowercase											=hidden_dropout_prob
      _lowercase											=attention_probs_dropout_prob
      _lowercase											=initializer_range
      _lowercase											=type_vocab_size
      _lowercase											=layer_norm_eps
      _lowercase											=use_cache

