






import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
  from tests.trainer.test_trainer import (  # noqa
      RegressionModelConfig,
      RegressionPreTrainedModel,
      get_regression_trainer,
  )


set_seed(42)

SCREAMING_SNAKE_CASE								=  get_tests_dir('fixtures')
SCREAMING_SNAKE_CASE								=  os.path.join(dirname(get_tests_dir()))
SCREAMING_SNAKE_CASE								=  dirname(os.path.abspath(__file__))

# default torch.distributed port
SCREAMING_SNAKE_CASE								=  '10999'

SCREAMING_SNAKE_CASE								=  't5-small'

# *** Working Models ***
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-albert'
SCREAMING_SNAKE_CASE								=  'sshleifer/bart-tiny-random'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-bert'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-bigbird_pegasus'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-big_bird'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-blenderbot'
SCREAMING_SNAKE_CASE								=  'bigscience/bigscience-small-testing'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-deberta'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-deberta-v2'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-distilbert-base-cased'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-electra'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-flaubert'
SCREAMING_SNAKE_CASE								=  'stas/tiny-wmt19-en-de'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-funnel'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-gpt2'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-gptj'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-gpt_neo'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-layoutlm'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-led'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-longformer'
SCREAMING_SNAKE_CASE								=  'stas/tiny-m2m_100'  # hf tiny model is unsuitable
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-marian-en-de'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-mbart'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-mobilebert'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-mpnet'
SCREAMING_SNAKE_CASE								=  'stas/pegasus-cnn_dailymail-tiny-random'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-prophetnet'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-distilroberta-base'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-squeezebert'
SCREAMING_SNAKE_CASE								=  'patrickvonplaten/t5-tiny-random'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-t5-v1.1'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-vit'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-xlm-roberta'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-xlnet-base-cased'


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-mt5'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-camembert'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-openai-gpt'

# missing tokenizer files
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-convbert'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-layoutlmv2'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-hubert'

# issues with tokenizer
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-ctrl'
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-transfo-xl'  # same as ctrl

# other issues with tiny models
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-ibert'  # multiple issues with either mlm/qa/clas
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-reformer'  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-dpr'
# - "dpr"  examples/research_projects/rag-end2end-retriever/
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-rag'
# - "rag" research_projects
SCREAMING_SNAKE_CASE								=  ''
# - "luke" Entities classes - no plan to make such example
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-lxmert'
# - "lxmert" doesn't work with run_qa.py
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-clip'
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-speech_to_text'
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
SCREAMING_SNAKE_CASE								=  'hf-internal-testing/tiny-random-tapas'

def        _lowerCamelCase		(  __A      :							str=False	) ->   List[Any]:
    # 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
    # - it won't be able to handle that
    # 2. for now testing with just 2 gpus max (since some quality tests may give different
    # results with mode gpus because we use very little data)
    _UpperCAmelCase			: str     					=			min(2						,						get_gpu_count()	) if distributed else 1
    _UpperCAmelCase			: Dict     					=			os.environ.get('''DS_TEST_PORT'''						,						__A	)
    return f'''deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}'''.split()

def        _lowerCamelCase		(  ) ->   Dict:
    _UpperCAmelCase			: str     					=			f'''{FIXTURE_DIRECTORY}/tests_samples'''
    _UpperCAmelCase			: int     					=			f'''{data_dir_samples}/wmt_en_ro'''
    _UpperCAmelCase			: Optional[int]     					=			f'''{data_dir_samples}/xsum'''
    _UpperCAmelCase			: Optional[int]     					=			'''
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        '''.split()

    # try to cover as many models as possible once (it's enough to run on one task per model)
    # but need a tiny model for each
    #
    # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
    _UpperCAmelCase			: int     					=			{
        '''trans''': [
            '''bart''',
            '''fsmt''',
            '''m2m_100''',
            '''marian''',
            '''mbart''',
            '''t5''',
            '''t5_v1''',
            # "mt5", missing model files
        ],
        '''sum''': [
            '''pegasus''',
        ],
        '''clm''': [
            '''big_bird''',
            '''bigbird_pegasus''',
            '''blenderbot''',
            '''bloom''',
            '''gpt2''',
            '''gpt_neo''',
            '''gptj''',
            '''xlm-roberta''',
            '''prophetnet''',
            # "camembert", missing model files
        ],
        '''mlm''': [
            '''albert''',
            '''deberta''',
            '''deberta-v2''',
            '''distilbert''',
            '''electra''',
            '''flaubert''',
            '''funnel''',
            '''layoutlm''',
            # "reformer", # multiple issues with either mlm/qa/clas
        ],
        '''qa''': [
            '''led''',
            '''longformer''',
            '''mobilebert''',
            '''mpnet''',
            '''roberta''',
            '''squeezebert''',
            # "convbert", # missing tokenizer files
            # "layoutlmv2", missing model files
        ],
        '''clas''': [
            '''bert''',
            '''xlnet''',
            # "hubert", # missing tokenizer files
            # "ibert", # multiple issues with either mlm/qa/clas
            # "transfo-xl", # tokenizer issues as ctrl
            # "ctrl", # tokenizer issues
            # "openai-gpt", missing model files
            # "tapas", multiple issues
        ],
        '''img_clas''': [
            '''vit''',
        ],
    }

    _UpperCAmelCase			: Optional[int]     					=			f'''{ROOT_DIRECTORY}/examples/pytorch'''

    _UpperCAmelCase			: List[str]     					=			{
        '''trans''': f'''
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        ''',
        '''sum''': f'''
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        ''',
        '''clm''': f'''
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        ''',
        '''mlm''': f'''
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        ''',
        '''qa''': f'''
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        ''',
        '''clas''': f'''
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        ''',
        '''img_clas''': f'''
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        ''',
    }

    _UpperCAmelCase			: Optional[Any]     					=			get_launcher(distributed=__A	)

    _UpperCAmelCase			: Optional[int]     					=			{}
    for task, args in tasks.items():
        _UpperCAmelCase			: List[Any]     					=			args.split()
        for model in tasksamodels[task]:
            _UpperCAmelCase			: str     					=			globals()[f'''{model.upper().replace('-'						,						'_'	)}_TINY''']
            _UpperCAmelCase			: Dict     					=			f'''--model_name_or_path {model_name}'''.split()
            _UpperCAmelCase			: str     					=			launcher + args + args_model + args_main

            # # generation special case
            # if task == "gen":
            #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
            #     args_model += f"--model_type {model}".split()
            #     cmds[f"{task}_{model}"] = launcher + args + args_model
            # else:

    return cmds


SCREAMING_SNAKE_CASE								=  make_task_cmds()

SCREAMING_SNAKE_CASE								=  'zero2'
SCREAMING_SNAKE_CASE								=  'zero3'

SCREAMING_SNAKE_CASE								=  [ZEROa, ZEROa]

def        _lowerCamelCase		(  __A      :							Optional[int]						,						__A      :							int						,						__A      :							int	) ->   List[str]:
    # customize the test name generator function as we want both params to appear in the sub-test
    # name, as by default it shows only the first param
    _UpperCAmelCase			: Union[str, Any]     					=			parameterized.to_safe_name('''_'''.join(str(__A	) for x in param.args	)	)
    return f'''{func.__name__}_{param_based_name}'''


# Cartesian-product of zero stages with models to test
SCREAMING_SNAKE_CASE								=  list(itertools.product(stages, task_cmds.keys()))


@slow
@require_deepspeed
@require_torch_gpu
class 							A_      (  __lowercase     ):






 '''simple docstring'''




 def 	snake_case__			(		self							,					_A							,					_A)	->						str:

     """simple docstring"""



     if task not in task_cmds:
         raise ValueError(f'''don\'t know of task {task}, have {task_cmds.keys()}''')

     _UpperCAmelCase			: str     					=			task_cmds[task]
     _UpperCAmelCase			: str     					=			f'''--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json'''.split()

     _UpperCAmelCase			: Optional[int]     					=			self.get_auto_remove_tmp_dir()
     _UpperCAmelCase			: List[str]     					=			f'''--output_dir {output_dir}'''.split()

     cmd += args_ds + args_out

     return cmd, output_dir


 @parameterized.expand(_A							,					name_func=_A)
 def 	snake_case__			(		self							,					_A							,					_A)	->						Tuple:

     """simple docstring"""



     _UpperCAmelCase					,	_UpperCAmelCase			: Tuple     					=			self.get_task_cmd(_A							,					_A)

     # 1. generate the checkpoint
     cmd += "--save_steps 1".split()
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
     execute_subprocess_async(_A							,					env=self.get_env())

     # 2. test that the fp32 weights get reconsolidated
     _UpperCAmelCase			: Tuple     					=			f'''{output_dir}/checkpoint-1'''
     _UpperCAmelCase			: Tuple     					=			f'''{chkpt_dir}/out.bin'''
     _UpperCAmelCase			: Optional[int]     					=			f'''{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}'''
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
     subprocess.check_call(_A							,					shell=_A)
     assert os.path.exists(_A), f'''{recovered_model_path} was not found'''

     # possibly could also test that the resulting saved model is usable but given that we use
     # random models we won't know if it's any good












import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)
class 							A_      :






 '''simple docstring'''




 def __init__(		self							,					_A)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: List[Any]     					=			foo


 def __call__(		self)	->						List[Any]:

     """simple docstring"""



     return self.foo



class 							A_      (  datasets.Dataset     ):






 '''simple docstring'''




 @fingerprint_transform(inplace=_A)
 def 	snake_case__			(		self							,					_A							,					*_A							,					**_A)	->						str:

     """simple docstring"""



     return DatasetChild(self.data							,					fingerprint=_A)


 @fingerprint_transform(inplace=_A)
 def 	snake_case__			(		self							,					_A							,					*_A							,					**_A)	->						Optional[Any]:

     """simple docstring"""



     return DatasetChild(self.data							,					fingerprint=_A)



class 							A_      :






 '''simple docstring'''




 def __init__(		self							,					_A)	->						List[str]:

     """simple docstring"""



     _UpperCAmelCase			: Union[str, Any]     					=			callable


 def __call__(		self							,					*_A							,					**_A)	->						Optional[int]:

     """simple docstring"""



     if self.callable is not None:
         return self.callable(*_A							,					**_A)


 def __getstate__(		self)	->						Optional[Any]:

     """simple docstring"""



     raise pickle.PicklingError()



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 @require_transformers
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     from transformers import AutoTokenizer

     def encode(_A):
         return tokenizer(_A)

     # TODO: add hash consistency tests across sessions
     _UpperCAmelCase			: Tuple     					=			AutoTokenizer.from_pretrained('''bert-base-uncased''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''bert-base-cased''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''bert-base-uncased''')
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_transformers
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Any:

     """simple docstring"""



     from transformers import AutoTokenizer

     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''gpt2''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     tokenizer('''Hello world !''')  # call once to change the tokenizer's cache
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)


 @require_regex
 def 	snake_case__			(		self)	->						Dict:

     """simple docstring"""



     import regex

     _UpperCAmelCase			: Optional[int]     					=			regex.Regex('''foo''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: int     					=			regex.Regex('''bar''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			regex.Regex('''foo''')
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     def func():
         return foo

     _UpperCAmelCase			: List[str]     					=			[0]
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			[1]
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Any     					=			[0]
     _UpperCAmelCase			: List[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						List[Any]:

     """simple docstring"""



     def func():
         pass

     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()

     def func():
         pass

     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: List[str]     					=			(
         '''co_argcount''',
         '''co_kwonlyargcount''',
         '''co_nlocals''',
         '''co_stacksize''',
         '''co_flags''',
         '''co_code''',
         '''co_consts''',
         '''co_names''',
         '''co_varnames''',
         '''co_filename''',
         '''co_name''',
         '''co_firstlineno''',
         '''co_lnotab''',
         '''co_freevars''',
         '''co_cellvars''',
     )

     def _create_code_pyaa(*_A):
         if hasattr(args[-3]							,					'''encode'''):
             _UpperCAmelCase			: Tuple     					=			list(_A)
             _UpperCAmelCase			: Optional[int]     					=			args[-3].encode()  # co_lnotab
             _UpperCAmelCase			: List[str]     					=			args[-10].encode()  # co_code
         if hasattr(_A							,					'''co_posonlyargcount'''):
             if len(_A) == 16:
                 return CodeType(*_A)
             elif len(_A) == 15:
                 return CodeType(args[0]							,					0							,					*args[1:])
             return CodeType(args[0]							,					0							,					0							,					*args[1:])
         elif hasattr(_A							,					'''co_kwonlyargcount'''):
             if len(_A) == 16:
                 return CodeType(args[0]							,					*args[2:])
             elif len(_A) == 15:
                 return CodeType(*_A)
             return CodeType(args[0]							,					0							,					*args[1:])
         if len(_A) == 16:
             return CodeType(args[0]							,					*args[3:])
         elif len(_A) == 15:
             return CodeType(args[0]							,					*args[2:])
         return CodeType(*_A)

     def create_ipython_func(_A							,					_A):
         def func():
             return returned_obj

         _UpperCAmelCase			: List[Any]     					=			func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse('''3.8'''):
             _UpperCAmelCase			: int     					=			_create_code_pyaa(
                 *[getattr(_A							,					_A) if k != '''co_filename''' else co_filename for k in code_args_pyaa])
         else:
             _UpperCAmelCase			: List[str]     					=			code.replace(co_filename=_A)
         return FunctionType(_A							,					func.__globals__							,					func.__name__							,					func.__defaults__							,					func.__closure__)

     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-2-e0383a102aae>''', [0]
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-2-e0383a102aae>''', [1]
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-5-713f6613acf3>''', [0]
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)

     _UpperCAmelCase					,	_UpperCAmelCase			: Union[str, Any]     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''321456789.py'''), [0]
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: str     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''321456789.py'''), [1]
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: Union[str, Any]     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''654123987.py'''), [0]
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase					,	_UpperCAmelCase			: List[str]     					=			[0], [1]

     def func():
         return foo, bar

     _UpperCAmelCase			: List[str]     					=			'''__main__'''

     def globalvars_mocka_side_effect(_A							,					*_A							,					**_A):
         return {"foo": foo, "bar": bar}

     def globalvars_mocka_side_effect(_A							,					*_A							,					**_A):
         return {"bar": bar, "foo": foo}

     with patch('''dill.detect.globalvars'''							,					side_effect=_A) as globalvars_mocka:
         _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count							,					0)
     with patch('''dill.detect.globalvars'''							,					side_effect=_A) as globalvars_mocka:
         _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count							,					0)
     self.assertEqual(_A							,					_A)



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 def 	snake_case__			(		self)	->						List[str]:

     """simple docstring"""



     _UpperCAmelCase			: Optional[int]     					=			Hasher.hash('''hello''')
     _UpperCAmelCase			: Union[str, Any]     					=			Hasher.hash('''hello''')
     _UpperCAmelCase			: Tuple     					=			Hasher.hash('''there''')
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: Union[str, Any]     					=			Hasher.hash(Foo('''hello'''))
     _UpperCAmelCase			: Optional[int]     					=			Hasher.hash(Foo('''hello'''))
     _UpperCAmelCase			: Any     					=			Hasher.hash(Foo('''there'''))
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Optional[int]:

     """simple docstring"""



     _UpperCAmelCase			: List[Any]     					=			Hasher()
     for x in ["hello", Foo('''hello''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[Any]     					=			hasher.hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			Hasher()
     for x in ["hello", Foo('''hello''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[int]     					=			hasher.hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			Hasher()
     for x in ["there", Foo('''there''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[int]     					=			hasher.hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     with self.assertRaises(pickle.PicklingError):
         Hasher.hash(UnpicklableCallable(Foo('''hello''')))


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: List[str]     					=			'''abc'''
     _UpperCAmelCase			: List[Any]     					=			[string, string]  # two strings have the same ids
     _UpperCAmelCase			: int     					=			[string, string]
     _UpperCAmelCase			: List[str]     					=			json.loads(f'''["{string}", "{string}"]''')  # two strings have different ids
     self.assertIs(obja[0]							,					_A)
     self.assertIs(obja[0]							,					obja[1])
     self.assertIs(obja[0]							,					_A)
     self.assertIs(obja[0]							,					obja[1])
     self.assertIsNot(obja[0]							,					_A)
     self.assertIsNot(obja[0]							,					obja[1])
     _UpperCAmelCase			: List[Any]     					=			Hasher.hash(_A)
     _UpperCAmelCase			: int     					=			Hasher.hash(_A)
     _UpperCAmelCase			: List[str]     					=			Hasher.hash(_A)
     self.assertEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)


 @require_tiktoken
 def 	snake_case__			(		self)	->						Any:

     """simple docstring"""



     import tiktoken

     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''gpt2''')
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''r50k_base''')
     _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''gpt2''')
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_torch
 def 	snake_case__			(		self)	->						Optional[int]:

     """simple docstring"""



     import torch

     _UpperCAmelCase			: Optional[int]     					=			torch.tensor([1.0])
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			torch.tensor([2.0])
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Any     					=			torch.tensor([1.0])
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_spacy
 @require_spacy_model('''en_core_web_sm''')
 @require_spacy_model('''fr_core_news_sm''')
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Tuple:

     """simple docstring"""



     import spacy

     _UpperCAmelCase			: Optional[int]     					=			spacy.load('''en_core_web_sm''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Dict     					=			spacy.load('''fr_core_news_sm''')
     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			spacy.load('''en_core_web_sm''')
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)



@pytest.mark.integration
def        _lowerCamelCase		(  __A      :							Path	) ->   int:
    _UpperCAmelCase			: Union[str, Any]     					=			tmp_path / '''dir1'''
    _UpperCAmelCase			: int     					=			tmp_path / '''dir2'''
    dira.mkdir()
    dira.mkdir()
    _UpperCAmelCase			: List[str]     					=			'''script.py'''
    _UpperCAmelCase			: str     					=			dedent(
        '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''	)
    _UpperCAmelCase			: Optional[Any]     					=			dira / script_filename
    _UpperCAmelCase			: List[Any]     					=			dira / script_filename
    with script_patha.open('''w'''	) as f:
        f.write(__A	)
    with script_patha.open('''w'''	) as f:
        f.write(__A	)
    _UpperCAmelCase			: Union[str, Any]     					=			subprocess.check_output(['''python''', str(__A	)]	)
    _UpperCAmelCase			: str     					=			subprocess.check_output(['''python''', str(__A	)]	)
    assert fingerprinta == fingerprinta

def        _lowerCamelCase		(  ) ->   Optional[int]:
    _UpperCAmelCase			: Union[str, Any]     					=			{'''a''': [0, 1, 2]}
    _UpperCAmelCase			: Union[str, Any]     					=			DatasetChild(InMemoryTable.from_pydict(__A	)	)
    _UpperCAmelCase			: Optional[Any]     					=			dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2	) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca	).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca	).get()._fingerprint

def        _lowerCamelCase		(  ) ->   str:
    _UpperCAmelCase			: str     					=			{'''a''': [0, 1, 2]}
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A)
     def 	snake_case__			(		self							,					_A)	->						Optional[int]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: Tuple     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A							,					version='''1.0.0''')
     def 	snake_case__			(		self							,					_A)	->						List[str]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: str     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A							,					version='''2.0.0''')
     def 	snake_case__			(		self							,					_A)	->						List[str]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: str     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}	) == 3


def        _lowerCamelCase		(  ) ->   List[str]:
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    _UpperCAmelCase			: Dict     					=			Hasher()
    hasher.update(lambda __A	: x	)




