
"""simple docstring"""




import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


_A      :	List[Any]       		=	namedtuple("""ExampleNamedTuple""", """a b c""")

class 		a__						(				unittest.TestCase						):
   def       __magic_name__      (      self    ):
          lowercase   :    List[Any]			=							torch.randn(5     ,   2    )
          lowercase   :    Optional[int]			=							torch.device("cuda"    ) if torch.cuda.is_available() else torch.device("cpu"    )

          lowercase   :    List[Any]			=							send_to_device(lowercase_     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta.cpu()     ,   lowercase_    )    )

          lowercase   :    Any			=							send_to_device((tensor, [tensor, tensor], 1)     ,   lowercase_    )
          self.assertIsInstance(lowercase_     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta[0].cpu()     ,   lowercase_    )    )
          self.assertIsInstance(resulta[1]     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta[1][0].cpu()     ,   lowercase_    )    )
          self.assertTrue(torch.equal(resulta[1][1].cpu()     ,   lowercase_    )    )
          self.assertEqual(resulta[2]     ,   1    )

          lowercase   :    Union[str, Any]			=							send_to_device({"a": tensor, "b": [tensor, tensor], "c": 1}     ,   lowercase_    )
          self.assertIsInstance(lowercase_     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta["a"].cpu()     ,   lowercase_    )    )
          self.assertIsInstance(resulta["b"]     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta["b"][0].cpu()     ,   lowercase_    )    )
          self.assertTrue(torch.equal(resulta["b"][1].cpu()     ,   lowercase_    )    )
          self.assertEqual(resulta["c"]     ,   1    )

          lowercase   :    Tuple			=							send_to_device(ExampleNamedTuple(a=lowercase_     ,   b=[tensor, tensor]     ,   c=1    )     ,   lowercase_    )
          self.assertIsInstance(lowercase_     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta.a.cpu()     ,   lowercase_    )    )
          self.assertIsInstance(resulta.b     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta.b[0].cpu()     ,   lowercase_    )    )
          self.assertTrue(torch.equal(resulta.b[1].cpu()     ,   lowercase_    )    )
          self.assertEqual(resulta.c     ,   1    )

          lowercase   :    Any			=							send_to_device(UserDict({"a": tensor, "b": [tensor, tensor], "c": 1}    )     ,   lowercase_    )
          self.assertIsInstance(lowercase_     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta["a"].cpu()     ,   lowercase_    )    )
          self.assertIsInstance(resulta["b"]     ,   lowercase_    )
          self.assertTrue(torch.equal(resulta["b"][0].cpu()     ,   lowercase_    )    )
          self.assertTrue(torch.equal(resulta["b"][1].cpu()     ,   lowercase_    )    )
          self.assertEqual(resulta["c"]     ,   1    )
   def       __magic_name__      (      self    ):
          with self.assertRaises(lowercase_    ) as cm:
                 lowercase   :    List[Any]			=							recursively_apply(torch.tensor     ,   (torch.tensor(1    ), 1)     ,   error_on_other_type=lowercase_    )
          self.assertEqual(
              str(cm.exception    )     ,   "Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed."     ,   )
   def       __magic_name__      (      self    ):
          lowercase   :    Tuple			=							torch.tensor([1, 2, 3, 4, 5]    )
          self.assertEqual(listify(lowercase_    )     ,   [1, 2, 3, 4, 5]    )

          lowercase   :    Union[str, Any]			=							torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]    )
          self.assertEqual(listify(lowercase_    )     ,   [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]    )

          lowercase   :    Tuple			=							torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]]    )
          self.assertEqual(
              listify(lowercase_    )     ,   [[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]]    )
   def       __magic_name__      (      self    ):
          with patch_environment(aa=1     ,   BB=2    ):
                 self.assertEqual(os.environ.get("AA"    )     ,   "1"    )
                 self.assertEqual(os.environ.get("BB"    )     ,   "2"    )

          self.assertNotIn("AA"     ,   os.environ    )
          self.assertNotIn("BB"     ,   os.environ    )
   def       __magic_name__      (      self    ):
          lowercase   :    Optional[Any]			=							RegressionModel()
          lowercase   :    Any			=							model.forward
          lowercase   :    List[str]			=							convert_outputs_to_fpaa(model.forward    )
          lowercase   :    List[str]			=							extract_model_from_parallel(lowercase_     ,   keep_fpaa_wrapper=lowercase_    )
          lowercase   :    Dict			=							pickle.dumps(lowercase_    )
   @require_cuda
   def       __magic_name__      (      self    ):
          lowercase   :    Optional[Any]			=							RegressionModel()
          lowercase   :    str			=							model.forward
          lowercase   :    Optional[Any]			=							torch.cuda.amp.autocast(dtype=torch.floataa    )(model.forward    )
          lowercase   :    Union[str, Any]			=							convert_outputs_to_fpaa(model.forward    )
          lowercase   :    List[str]			=							extract_model_from_parallel(lowercase_     ,   keep_fpaa_wrapper=lowercase_    )
          lowercase   :    Dict			=							pickle.dumps(lowercase_    )
   @require_cuda
   @require_torch_min_version(version="2.0"    )
   def       __magic_name__      (      self    ):
          lowercase   :    int			=							RegressionModel()
          lowercase   :    Tuple			=							model.forward
          lowercase   :    List[Any]			=							torch.cuda.amp.autocast(dtype=torch.floataa    )(model.forward    )
          lowercase   :    Union[str, Any]			=							convert_outputs_to_fpaa(model.forward    )
          lowercase   :    Any			=							torch.compile(model.forward     ,   backend="inductor"    )
          lowercase   :    Union[str, Any]			=							torch.randn(4     ,   10    ).cuda()
          lowercase   :    Optional[Any]			=							model(lowercase_    )
   def       __magic_name__      (      self    ):
          lowercase   :    Any			=							RegressionModel()
          # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
          lowercase   :    Optional[int]			=							torch.nn.parallel.DataParallel(lowercase_    )
          lowercase   :    Dict			=							extract_model_from_parallel(lowercase_    )

          self.assertEqual(lowercase_     ,   lowercase_    )
   @require_torch_min_version(version="2.0"    )
   def       __magic_name__      (      self    ):
          lowercase   :    List[Any]			=							RegressionModel()
          lowercase   :    Union[str, Any]			=							torch.compile(lowercase_    )

          # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
          lowercase   :    str			=							torch.nn.parallel.DataParallel(lowercase_    )
          lowercase   :    Union[str, Any]			=							torch.compile(lowercase_    )
          lowercase   :    Any			=							extract_model_from_parallel(lowercase_    )

          self.assertEqual(compiled_model._orig_mod     ,   compiled_model_unwrapped._orig_mod    )

   def       __magic_name__      (      self    ):
          self.assertEqual(find_device([1, "a", torch.tensor([1, 2, 3]    )]    )     ,   torch.device("cpu"    )    )
          self.assertEqual(find_device({"a": 1, "b": torch.tensor([1, 2, 3]    )}    )     ,   torch.device("cpu"    )    )
          self.assertIsNone(find_device([1, "a"]    )    )




"""simple docstring"""




import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 		a__						(				a_,      unittest.TestCase						):
   __lowerCAmelCase						=    BertweetTokenizer
   __lowerCAmelCase						=    False
   def       __magic_name__      (      self    ):
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          lowercase   :    Tuple			=							["I", "m", "V@@", "R@@", "r", "e@@"]
          lowercase   :    List[Any]			=							dict(zip(_a     ,   range(len(_a    )    )    )    )
          lowercase   :    Union[str, Any]			=							["#version: 0.2", "a m</w>"]
          lowercase   :    List[Any]			=							{"unk_token": "<unk>"}

          lowercase   :    Optional[Any]			=							os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["vocab_file"]    )
          lowercase   :    Any			=							os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["merges_file"]    )
          with open(self.vocab_file     ,   "w"     ,   encoding="utf-8"    ) as fp:
                 for token in vocab_tokens:
                        fp.write(f"""{token} {vocab_tokens[token]}\n"""    )
          with open(self.merges_file     ,   "w"     ,   encoding="utf-8"    ) as fp:
                 fp.write("\n".join(_a    )    )
   def       __magic_name__      (      self     ,   **_a    ):
          kwargs.update(self.special_tokens_map    )
          return BertweetTokenizer.from_pretrained(self.tmpdirname     ,   **_a    )
   def       __magic_name__      (      self     ,   _a    ):
          lowercase   :    Union[str, Any]			=							"I am VinAI Research"
          lowercase   :    str			=							"I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
          return input_text, output_text

   def       __magic_name__      (      self    ):
          lowercase   :    List[Any]			=							BertweetTokenizer(self.vocab_file     ,   self.merges_file     ,   **self.special_tokens_map    )
          lowercase   :    int			=							"I am VinAI Research"
          lowercase   :    Dict			=							"I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
          lowercase   :    Tuple			=							tokenizer.tokenize(_a    )
          self.assertListEqual(_a     ,   _a    )

          lowercase   :    Any			=							tokens + [tokenizer.unk_token]

          lowercase   :    List[Any]			=							[4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_a    )     ,   _a    )


