





'''simple docstring'''


import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class 		__lowercase			:

			def __init__(self     ,				A ):
								lowerCamelCase_   : Union[str, Any]       =						foo






			def __call__(self ):
								return self.foo

class 		__lowercase			(					datasets.Dataset      ):

			@fingerprint_transform(inplace=A )
			def     UpperCAmelCase__	(self     ,				A     ,				*A     ,				**A ):
								return DatasetChild(self.data     ,				fingerprint=A )






			@fingerprint_transform(inplace=A )
			def     UpperCAmelCase__	(self     ,				A     ,				*A     ,				**A ):
								return DatasetChild(self.data     ,				fingerprint=A )

class 		__lowercase			:

			def __init__(self     ,				A ):
								lowerCamelCase_   : Any       =						callable

			def __call__(self     ,				*A     ,				**A ):







								if self.callable is not None:
													return self.callable(*A     ,				**A )






			def __getstate__(self ):
								raise pickle.PicklingError()

class 		__lowercase			(					lowercase_      ):

			@require_transformers
			@pytest.mark.integration
			def     UpperCAmelCase__	(self ):
								from transformers import AutoTokenizer

								def encode(A ):
													return tokenizer(A )

								# TODO: add hash consistency tests across sessions
								lowerCamelCase_   : int       =						AutoTokenizer.from_pretrained('''bert-base-uncased''' )
								lowerCamelCase_   : Optional[int]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(lambda A : tokenizer(A ) ) ).hexdigest()
								lowerCamelCase_   : Union[str, Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : str       =						AutoTokenizer.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Optional[Any]       =						mda(datasets.utils.py_utils.dumps(lambda A : tokenizer(A ) ) ).hexdigest()
								lowerCamelCase_   : List[str]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Optional[int]       =						AutoTokenizer.from_pretrained('''bert-base-uncased''' )
								lowerCamelCase_   : List[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Union[str, Any]       =						mda(datasets.utils.py_utils.dumps(lambda A : tokenizer(A ) ) ).hexdigest()
								lowerCamelCase_   : Optional[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			@require_transformers
			@pytest.mark.integration
			def     UpperCAmelCase__	(self ):
								from transformers import AutoTokenizer

								lowerCamelCase_   : str       =						AutoTokenizer.from_pretrained('''gpt2''' )
								lowerCamelCase_   : Optional[int]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								tokenizer('''Hello world !''' )  # call once to change the tokenizer's cache
								lowerCamelCase_   : Tuple       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )






			@require_regex
			def     UpperCAmelCase__	(self ):
								import regex

								lowerCamelCase_   : Tuple       =						regex.Regex('''foo''' )
								lowerCamelCase_   : Optional[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Optional[int]       =						regex.Regex('''bar''' )
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : str       =						regex.Regex('''foo''' )
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

class 		__lowercase			(					lowercase_      ):

			def     UpperCAmelCase__	(self ):







								def func():
													return foo

								lowerCamelCase_   : int       =						[0]
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : List[str]       =						[1]
								lowerCamelCase_   : Tuple       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : str       =						[0]
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):







								def func():
													pass

								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()

								def func():
													pass

								lowerCamelCase_   : Optional[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : List[str]       =						mda(datasets.utils.py_utils.dumps(Foo([0] ) ) ).hexdigest()
								lowerCamelCase_   : Union[str, Any]       =						mda(datasets.utils.py_utils.dumps(Foo([1] ) ) ).hexdigest()
								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(Foo([0] ) ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Dict       =						mda(datasets.utils.py_utils.dumps(Foo([0] ).__call__ ) ).hexdigest()
								lowerCamelCase_   : Dict       =						mda(datasets.utils.py_utils.dumps(Foo([1] ).__call__ ) ).hexdigest()
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(Foo([0] ).__call__ ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[int]       =						(
								    '''co_argcount''',
								    '''co_kwonlyargcount''',
								    '''co_nlocals''',
								    '''co_stacksize''',
								    '''co_flags''',
								    '''co_code''',
								    '''co_consts''',
								    '''co_names''',
								    '''co_varnames''',
								    '''co_filename''',
								    '''co_name''',
								    '''co_firstlineno''',
								    '''co_lnotab''',
								    '''co_freevars''',
								    '''co_cellvars''',
								)

								def _create_code_pyaa(*A ):
													if hasattr(args[-3]     ,				'''encode''' ):
																		lowerCamelCase_   : Tuple       =						list(A )
																		lowerCamelCase_   : str       =						args[-3].encode()  # co_lnotab
																		lowerCamelCase_   : Any       =						args[-1_0].encode()  # co_code
													if hasattr(A     ,				'''co_posonlyargcount''' ):
																		if len(A ) == 1_6:
																							return CodeType(*A )
																		elif len(A ) == 1_5:
																							return CodeType(args[0]     ,				0     ,				*args[1:] )
																		return CodeType(args[0]     ,				0     ,				0     ,				*args[1:] )
													elif hasattr(A     ,				'''co_kwonlyargcount''' ):
																		if len(A ) == 1_6:
																							return CodeType(args[0]     ,				*args[2:] )
																		elif len(A ) == 1_5:
																							return CodeType(*A )
																		return CodeType(args[0]     ,				0     ,				*args[1:] )
													if len(A ) == 1_6:
																		return CodeType(args[0]     ,				*args[3:] )
													elif len(A ) == 1_5:
																		return CodeType(args[0]     ,				*args[2:] )
													return CodeType(*A )

								def create_ipython_func(A     ,				A ):
													def func():
																		return returned_obj

													lowerCamelCase_   : Union[str, Any]       =						func.__code__
													# Use _create_code from dill in order to make it work for different python versions
													if datasets.config.PY_VERSION < version.parse('''3.8''' ):
																		lowerCamelCase_   : str       =						_create_code_pyaa(
																		    *[getattr(A     ,				A ) if k != '''co_filename''' else co_filename for k in code_args_pyaa] )
													else:
																		lowerCamelCase_   : Dict       =						code.replace(co_filename=A )
													return FunctionType(A     ,				func.__globals__     ,				func.__name__     ,				func.__defaults__     ,				func.__closure__ )

								lowerCamelCase_, lowerCamelCase_   : List[Any]       =						'''<ipython-input-2-e0383a102aae>''', [0]
								lowerCamelCase_   : Dict       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								lowerCamelCase_, lowerCamelCase_   : Optional[int]       =						'''<ipython-input-2-e0383a102aae>''', [1]
								lowerCamelCase_   : List[str]       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								lowerCamelCase_, lowerCamelCase_   : Optional[int]       =						'''<ipython-input-5-713f6613acf3>''', [0]
								lowerCamelCase_   : Dict       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

								lowerCamelCase_, lowerCamelCase_   : List[Any]       =						os.path.join(gettempdir()     ,				'''ipykernel_12345'''     ,				'''321456789.py''' ), [0]
								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								lowerCamelCase_, lowerCamelCase_   : Union[str, Any]       =						os.path.join(gettempdir()     ,				'''ipykernel_12345'''     ,				'''321456789.py''' ), [1]
								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								lowerCamelCase_, lowerCamelCase_   : Union[str, Any]       =						os.path.join(gettempdir()     ,				'''ipykernel_12345'''     ,				'''654123987.py''' ), [0]
								lowerCamelCase_   : Tuple       =						mda(datasets.utils.py_utils.dumps(create_ipython_func(A     ,				A ) ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )






			def     UpperCAmelCase__	(self ):
								lowerCamelCase_, lowerCamelCase_   : List[str]       =						[0], [1]

								def func():
													return foo, bar

								lowerCamelCase_   : Optional[int]       =						'''__main__'''

								def globalvars_mocka_side_effect(A     ,				*A     ,				**A ):
													return {"foo": foo, "bar": bar}

								def globalvars_mocka_side_effect(A     ,				*A     ,				**A ):
													return {"bar": bar, "foo": foo}

								with patch('''dill.detect.globalvars'''     ,				side_effect=A ) as globalvars_mocka:
													lowerCamelCase_   : List[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
													self.assertGreater(globalvars_mocka.call_count     ,				0 )
								with patch('''dill.detect.globalvars'''     ,				side_effect=A ) as globalvars_mocka:
													lowerCamelCase_   : Optional[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
													self.assertGreater(globalvars_mocka.call_count     ,				0 )
								self.assertEqual(A     ,				A )

class 		__lowercase			(					lowercase_      ):

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : int       =						Hasher.hash('''hello''' )
								lowerCamelCase_   : str       =						Hasher.hash('''hello''' )
								lowerCamelCase_   : Union[str, Any]       =						Hasher.hash('''there''' )
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : List[str]       =						Hasher.hash(Foo('''hello''' ) )
								lowerCamelCase_   : List[str]       =						Hasher.hash(Foo('''hello''' ) )
								lowerCamelCase_   : str       =						Hasher.hash(Foo('''there''' ) )
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[int]       =						Hasher()
								for x in ["hello", Foo('''hello''' )]:
													hasher.update(A )
								lowerCamelCase_   : Optional[int]       =						hasher.hexdigest()
								lowerCamelCase_   : int       =						Hasher()
								for x in ["hello", Foo('''hello''' )]:
													hasher.update(A )
								lowerCamelCase_   : Optional[int]       =						hasher.hexdigest()
								lowerCamelCase_   : Dict       =						Hasher()
								for x in ["there", Foo('''there''' )]:
													hasher.update(A )
								lowerCamelCase_   : Union[str, Any]       =						hasher.hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			def     UpperCAmelCase__	(self ):







								with self.assertRaises(pickle.PicklingError ):
													Hasher.hash(UnpicklableCallable(Foo('''hello''' ) ) )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[int]       =						'''abc'''
								lowerCamelCase_   : Tuple       =						[string, string]  # two strings have the same ids
								lowerCamelCase_   : Union[str, Any]       =						[string, string]
								lowerCamelCase_   : Optional[Any]       =						json.loads(F"""[\"{string}\", \"{string}\"]""" )  # two strings have different ids
								self.assertIs(obja[0]     ,				A )
								self.assertIs(obja[0]     ,				obja[1] )
								self.assertIs(obja[0]     ,				A )
								self.assertIs(obja[0]     ,				obja[1] )
								self.assertIsNot(obja[0]     ,				A )
								self.assertIsNot(obja[0]     ,				obja[1] )
								lowerCamelCase_   : List[str]       =						Hasher.hash(A )
								lowerCamelCase_   : Any       =						Hasher.hash(A )
								lowerCamelCase_   : str       =						Hasher.hash(A )
								self.assertEqual(A     ,				A )
								self.assertEqual(A     ,				A )

			@require_tiktoken
			def     UpperCAmelCase__	(self ):
								import tiktoken

								lowerCamelCase_   : List[str]       =						tiktoken.get_encoding('''gpt2''' )
								lowerCamelCase_   : List[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : List[str]       =						tiktoken.get_encoding('''r50k_base''' )
								lowerCamelCase_   : Optional[int]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : List[str]       =						tiktoken.get_encoding('''gpt2''' )
								lowerCamelCase_   : List[Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )

			@require_torch
			def     UpperCAmelCase__	(self ):
								import torch

								lowerCamelCase_   : Optional[int]       =						torch.tensor([1.0] )
								lowerCamelCase_   : int       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Tuple       =						torch.tensor([2.0] )
								lowerCamelCase_   : Optional[int]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : Any       =						torch.tensor([1.0] )
								lowerCamelCase_   : Union[str, Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )






			@require_spacy
			@require_spacy_model('''en_core_web_sm''' )
			@require_spacy_model('''fr_core_news_sm''' )
			@pytest.mark.integration
			def     UpperCAmelCase__	(self ):
								import spacy

								lowerCamelCase_   : Any       =						spacy.load('''en_core_web_sm''' )
								lowerCamelCase_   : str       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : int       =						spacy.load('''fr_core_news_sm''' )
								lowerCamelCase_   : Union[str, Any]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								lowerCamelCase_   : str       =						spacy.load('''en_core_web_sm''' )
								lowerCamelCase_   : Optional[int]       =						mda(datasets.utils.py_utils.dumps(A ) ).hexdigest()
								self.assertEqual(A     ,				A )
								self.assertNotEqual(A     ,				A )




@pytest.mark.integration
def lowercase_		(				_lowercase			)       ->							Any:
					'''simple docstring'''

					lowerCamelCase_   : Optional[Any]       =						tmp_path / '''dir1'''
					lowerCamelCase_   : str       =						tmp_path / '''dir2'''
					dira.mkdir()
					dira.mkdir()
					lowerCamelCase_   : Optional[int]       =						'''script.py'''
					lowerCamelCase_   : Any       =						dedent(
					    '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''			)
					lowerCamelCase_   : List[str]       =						dira / script_filename
					lowerCamelCase_   : Optional[Any]       =						dira / script_filename
					with script_patha.open('''w'''			) as f:
										f.write(_lowercase			)
					with script_patha.open('''w'''			) as f:
										f.write(_lowercase			)
					lowerCamelCase_   : List[str]       =						subprocess.check_output(['''python''', str(_lowercase			)]			)
					lowerCamelCase_   : int       =						subprocess.check_output(['''python''', str(_lowercase			)]			)
					assert fingerprinta == fingerprinta




def lowercase_		(				)       ->							List[Any]:
					'''simple docstring'''

					lowerCamelCase_   : Tuple       =						{'''a''': [0, 1, 2]}
					lowerCamelCase_   : List[Any]       =						DatasetChild(InMemoryTable.from_pydict(_lowercase			)			)
					lowerCamelCase_   : Dict       =						dataset.funca()._fingerprint
					assert expected_fingerprint == dataset.funca()._fingerprint
					assert expected_fingerprint != dataset.funca()._fingerprint

					with Pool(2			) as p:
										assert expected_fingerprint == p.apply_async(dataset.funca			).get()._fingerprint
										assert expected_fingerprint != p.apply_async(dataset.funca			).get()._fingerprint




def lowercase_		(				)       ->							List[str]:
					'''simple docstring'''

					lowerCamelCase_   : List[Any]       =						{'''a''': [0, 1, 2]}

					class 		__lowercase			(					datasets.Dataset      ):

								@fingerprint_transform(inplace=A )
								def     UpperCAmelCase__	(self     ,				A ):
													return DummyDatasetChild(self.data     ,				fingerprint=A )
					lowerCamelCase_   : Union[str, Any]       =						DummyDatasetChild(InMemoryTable.from_pydict(_lowercase			)			).func()

					class 		__lowercase			(					datasets.Dataset      ):

								@fingerprint_transform(inplace=A     ,				version='''1.0.0''' )
								def     UpperCAmelCase__	(self     ,				A ):
													return DummyDatasetChild(self.data     ,				fingerprint=A )
					lowerCamelCase_   : Optional[Any]       =						DummyDatasetChild(InMemoryTable.from_pydict(_lowercase			)			).func()

					class 		__lowercase			(					datasets.Dataset      ):

								@fingerprint_transform(inplace=A     ,				version='''2.0.0''' )
								def     UpperCAmelCase__	(self     ,				A ):
													return DummyDatasetChild(self.data     ,				fingerprint=A )
					lowerCamelCase_   : int       =						DummyDatasetChild(InMemoryTable.from_pydict(_lowercase			)			).func()

					assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}			) == 3






def lowercase_		(				)       ->							Any:
					'''simple docstring'''

					lowerCamelCase_   : Any       =						Hasher()
					hasher.update(lambda _lowercase			: x			)




'''simple docstring'''


import argparse
import logging
import sys
from unittest.mock import patch

import run_glue_with_pabee

from transformers.testing_utils import TestCasePlus


logging.basicConfig(level=logging.DEBUG)

__lowercase							:				Optional[Any]					=       logging.getLogger()




def lowercase_		(				)       ->							Union[str, Any]:
					'''simple docstring'''

					lowerCamelCase_   : Any       =						argparse.ArgumentParser()
					parser.add_argument('''-f'''			)
					lowerCamelCase_   : Optional[int]       =						parser.parse_args()
					return args.f






class 		__lowercase			(					_lowercase      ):

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Dict       =						logging.StreamHandler(sys.stdout )
								logger.addHandler(A )

								lowerCamelCase_   : int       =						self.get_auto_remove_tmp_dir()
								lowerCamelCase_   : Dict       =						F"""
            run_glue_with_pabee.py
            --model_type albert
            --model_name_or_path albert-base-v2
            --data_dir ./tests/fixtures/tests_samples/MRPC/
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --task_name mrpc
            --do_train
            --do_eval
            --per_gpu_train_batch_size=2
            --per_gpu_eval_batch_size=1
            --learning_rate=2e-5
            --max_steps=50
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

								with patch.object(A     ,				'''argv'''     ,				A ):
													lowerCamelCase_   : Optional[Any]       =						run_glue_with_pabee.main()
													for value in result.values():
																		self.assertGreaterEqual(A     ,				0.75 )


