






import argparse
import logging
import os
import sys
import tempfile
from pathlib import Path

import lightning_base
import pytest
import pytorch_lightning as pl
import torch
from convert_pl_checkpoint_to_hf import convert_pl_to_hf
from distillation import distill_main
from finetune import SummarizationModule, main
from huggingface_hub import list_models
from parameterized import parameterized
from run_eval import generate_summaries_or_translations
from torch import nn

from transformers import AutoConfig, AutoModelForSeqaSeqLM
from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
from utils import label_smoothed_nll_loss, lmap, load_json


logging.basicConfig(level=logging.DEBUG)

SCREAMING_SNAKE_CASE								=  logging.getLogger()
SCREAMING_SNAKE_CASE								=  torch.cuda.is_available()
SCREAMING_SNAKE_CASE								=  {
    'max_tokens_per_batch': None,
    'supervise_forward': True,
    'normalize_hidden': True,
    'label_smoothing': 0.2,
    'eval_max_gen_length': None,
    'eval_beams': 1,
    'val_metric': 'loss',
    'save_top_k': 1,
    'adafactor': True,
    'early_stopping_patience': 2,
    'logger_name': 'default',
    'length_penalty': 0.5,
    'cache_dir': '',
    'task': 'summarization',
    'num_workers': 2,
    'alpha_hid': 0,
    'freeze_embeds': True,
    'enc_only': False,
    'tgt_suffix': '',
    'resume_from_checkpoint': None,
    'sortish_sampler': True,
    'student_decoder_layers': 1,
    'val_check_interval': 1.0,
    'output_dir': '',
    'fp16': False,  # TODO(SS): set this to CUDA_AVAILABLE if ci installs apex or start using native amp
    'no_teacher': False,
    'fp16_opt_level': 'O1',
    'gpus': 1 if CUDA_AVAILABLE else 0,
    'n_tpu_cores': 0,
    'max_grad_norm': 1.0,
    'do_train': True,
    'do_predict': True,
    'accumulate_grad_batches': 1,
    'server_ip': '',
    'server_port': '',
    'seed': 42,
    'model_name_or_path': 'sshleifer/bart-tiny-random',
    'config_name': '',
    'tokenizer_name': 'facebook/bart-large',
    'do_lower_case': False,
    'learning_rate': 0.3,
    'lr_scheduler': 'linear',
    'weight_decay': 0.0,
    'adam_epsilon': 1e-0_8,
    'warmup_steps': 0,
    'max_epochs': 1,
    'train_batch_size': 2,
    'eval_batch_size': 2,
    'max_source_length': 12,
    'max_target_length': 12,
    'val_max_target_length': 12,
    'test_max_target_length': 12,
    'fast_dev_run': False,
    'no_cache': False,
    'n_train': -1,
    'n_val': -1,
    'n_test': -1,
    'student_encoder_layers': 1,
    'freeze_encoder': False,
    'auto_scale_batch_size': False,
    'overwrite_output_dir': False,
    'student': None,
}

def        _lowerCamelCase		(  __A      :							Path						,						__A      :							list	) ->   Any:
    _UpperCAmelCase			: Any     					=			'''\n'''.join(__A	)
    Path(__A	).open('''w'''	).writelines(__A	)


SCREAMING_SNAKE_CASE								=  [' Sam ate lunch today.', 'Sams lunch ingredients.']
SCREAMING_SNAKE_CASE								=  ['A very interesting story about what I ate for lunch.', 'Avocado, celery, turkey, coffee']
SCREAMING_SNAKE_CASE								=  'patrickvonplaten/t5-tiny-random'
SCREAMING_SNAKE_CASE								=  'sshleifer/t5-tinier-random'
SCREAMING_SNAKE_CASE								=  'sshleifer/bart-tiny-random'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-mbart'
SCREAMING_SNAKE_CASE								=  'sshleifer/tiny-marian-en-de'
SCREAMING_SNAKE_CASE								=  'stas/tiny-wmt19-en-de'


SCREAMING_SNAKE_CASE								=  logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks

def        _lowerCamelCase		(  __A      :							List[Any]	) ->   Optional[Any]:
    for split in ["train", "val", "test"]:
        _dump_articles(os.path.join(__A						,						f'''{split}.source'''	)						,						__A	)
        _dump_articles(os.path.join(__A						,						f'''{split}.target'''	)						,						__A	)
    return tmp_dir
class 							A_      (  __lowercase     ):






 '''simple docstring'''




 @classmethod
 def 	snake_case__			(		cls)	->						List[Any]:

     """simple docstring"""



     logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
     return cls


 @slow
 @require_torch_gpu
 def 	snake_case__			(		self)	->						str:

     """simple docstring"""



     _UpperCAmelCase			: Dict     					=			list_models()
     _UpperCAmelCase			: int     					=			'''sshleifer'''
     _UpperCAmelCase			: Union[str, Any]     					=			[x.modelId for x in model_list if x.modelId.startswith(_A)]
     _UpperCAmelCase			: Tuple     					=			['''sshleifer/blenderbot-3B''', '''sshleifer/blenderbot-90M''']
     _UpperCAmelCase			: Optional[int]     					=			[]
     for m in model_ids:
         if m in allowed_to_be_broken:
             continue
         try:
             AutoConfig.from_pretrained(_A)
         except Exception:
             failures.append(_A)
     assert not failures, f'''The following models could not be loaded through AutoConfig: {failures}'''


 def 	snake_case__			(		self)	->						Tuple:

     """simple docstring"""



     _UpperCAmelCase			: Optional[int]     					=			{'''student_encoder_layers''': 2, '''student_decoder_layers''': 1, '''no_teacher''': True}
     self._test_distiller_cli(_A)


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: Any     					=			{
         '''student_encoder_layers''': 2,
         '''student_decoder_layers''': 1,
         '''max_epochs''': 4,
         '''val_check_interval''': 0.25,
         '''alpha_hid''': 2.0,
         '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
     }
     _UpperCAmelCase			: List[Any]     					=			self._test_distiller_cli(_A							,					check_contents=_A)

     _UpperCAmelCase			: Optional[int]     					=			list(Path(model.output_dir).glob('''*.ckpt'''))
     self.assertEqual(1							,					len(_A))
     _UpperCAmelCase			: Any     					=			list(Path(model.output_dir).glob('''**/*.bin'''))
     self.assertEqual(len(_A)							,					2)
     _UpperCAmelCase			: str     					=			lmap(str.strip							,					Path(model.hparams.data_dir).joinpath('''test.source''').open().readlines())
     _UpperCAmelCase			: Tuple     					=			tempfile.mktemp()  # XXX: not being cleaned up
     generate_summaries_or_translations(_A							,					_A							,					str(model.output_dir / '''best_tfmr'''))
     self.assertTrue(Path(_A).exists())

     _UpperCAmelCase			: List[str]     					=			self.get_auto_remove_tmp_dir()
     convert_pl_to_hf(ckpts[0]							,					transformer_ckpts[0].parent							,					_A)
     assert os.path.exists(os.path.join(_A							,					'''pytorch_model.bin'''))


 def 	snake_case__			(		self)	->						Any:

     """simple docstring"""



     _UpperCAmelCase			: int     					=			AutoModelForSeqaSeqLM.from_pretrained(_A)
     _UpperCAmelCase			: Optional[int]     					=			model.dummy_inputs['''input_ids'''], model.dummy_inputs['''attention_mask''']
     _UpperCAmelCase			: List[str]     					=			torch.tensor([[0, 4, 8, 2], [0, 8, 2, 1]]							,					dtype=torch.long							,					device=model.device)
     _UpperCAmelCase			: Optional[Any]     					=			target_ids[:, :-1].contiguous()  # Why this line?
     _UpperCAmelCase			: int     					=			target_ids[:, 1:].clone()  # why clone?
     _UpperCAmelCase			: Any     					=			model(
         _A							,					attention_mask=_A							,					decoder_input_ids=_A							,					labels=_A							,					use_cache=_A).loss

     _UpperCAmelCase			: str     					=			model(_A							,					attention_mask=_A							,					decoder_input_ids=_A							,					use_cache=_A).logits

     _UpperCAmelCase			: Any     					=			nn.functional.log_softmax(_A							,					dim=-1)
     _UpperCAmelCase			: Any     					=			label_smoothed_nll_loss(
         _A							,					_A							,					0.1							,					ignore_index=model.config.pad_token_id)
     with self.assertRaises(_A):
         # TODO: understand why this breaks
         self.assertEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase			: Optional[int]     					=			{
         '''student_encoder_layers''': 2,
         '''student_decoder_layers''': 1,
         '''num_train_epochs''': 4,
         '''val_check_interval''': 0.25,
         '''alpha_hid''': 2.0,
         '''task''': '''translation''',
         '''model_name_or_path''': '''IGNORE_THIS_IT_DOESNT_GET_USED''',
         '''tokenizer_name''': MBART_TINY,
         '''teacher''': MBART_TINY,
         '''src_lang''': '''en_XX''',
         '''tgt_lang''': '''ro_RO''',
     }
     _UpperCAmelCase			: List[str]     					=			self._test_distiller_cli(_A							,					check_contents=_A)
     assert model.model.config.model_type == "mbart"

     _UpperCAmelCase			: List[str]     					=			list(Path(model.output_dir).glob('''*.ckpt'''))
     self.assertEqual(1							,					len(_A))
     _UpperCAmelCase			: Dict     					=			list(Path(model.output_dir).glob('''**/*.bin'''))
     _UpperCAmelCase			: Optional[int]     					=			list(Path(model.output_dir).glob('''best_tfmr/*'''))
     assert len(_A) > 2
     self.assertEqual(len(_A)							,					2)


 def 	snake_case__			(		self)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: Optional[int]     					=			{
         '''student_encoder_layers''': 1,
         '''student_decoder_layers''': 1,
         '''alpha_hid''': 2.0,
         '''teacher''': T5_TINY,
         '''model_name_or_path''': T5_TINY,
         '''tokenizer_name''': T5_TINY,
     }
     self._test_distiller_cli(_A)


 def 	snake_case__			(		self)	->						Tuple:

     """simple docstring"""



     _UpperCAmelCase			: Any     					=			{
         '''teacher''': T5_TINY,
         '''student''': T5_TINIER,
         '''model_name_or_path''': T5_TINIER,
         '''tokenizer_name''': T5_TINIER,
     }
     self._test_distiller_cli(_A)


 def 	snake_case__			(		self							,					_A							,					_A=True)	->						Any:

     """simple docstring"""



     _UpperCAmelCase			: List[str]     					=			{
         '''label_smoothing''': 0.0,
         '''early_stopping_patience''': -1,
         '''train_batch_size''': 1,
         '''eval_batch_size''': 2,
         '''max_epochs''': 2,
         '''alpha_mlm''': 0.2,
         '''alpha_ce''': 0.8,
         '''do_predict''': True,
         '''model_name_or_path''': '''sshleifer/tinier_bart''',
         '''teacher''': CHEAP_ARGS['''model_name_or_path'''],
         '''val_check_interval''': 0.5,
     }
     default_updates.update(_A)
     _UpperCAmelCase			: dict     					=			CHEAP_ARGS.copy()
     _UpperCAmelCase			: Tuple     					=			make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
     _UpperCAmelCase			: Union[str, Any]     					=			self.get_auto_remove_tmp_dir()

     args_d.update(data_dir=_A							,					output_dir=_A							,					**_A)
     _UpperCAmelCase			: Union[str, Any]     					=			distill_main(argparse.Namespace(**_A))
     if not check_contents:
         return model
     _UpperCAmelCase			: str     					=			os.listdir(_A)
     _UpperCAmelCase			: str     					=			{os.path.basename(_A) for p in contents}
     _UpperCAmelCase			: Tuple     					=			[p for p in contents if p.endswith('''ckpt''')]
     assert len(_A) > 0

     self.assertIn('''test_generations.txt'''							,					_A)
     self.assertIn('''test_results.txt'''							,					_A)

     _UpperCAmelCase			: List[str]     					=			load_json(model.metrics_save_path)
     _UpperCAmelCase			: int     					=			metrics['''val'''][-1]
     self.assertGreaterEqual(last_step_stats['''val_avg_gen_time''']							,					0.01)
     self.assertGreaterEqual(1.0							,					last_step_stats['''val_avg_gen_time'''])
     self.assertIsInstance(last_step_stats[f'''val_avg_{model.val_metric}''']							,					_A)
     _UpperCAmelCase			: Any     					=			int(args_d['''max_epochs'''] * (1 / args_d['''val_check_interval''']) + 1)
     self.assertEqual(len(metrics['''val'''])							,					_A)
     self.assertEqual(len(metrics['''test'''])							,					1)
     return model



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 @parameterized.expand(
     [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY]							,					)
 def 	snake_case__			(		self							,					_A)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: dict     					=			CHEAP_ARGS.copy()
     _UpperCAmelCase			: Tuple     					=			'''translation''' if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else '''summarization'''
     _UpperCAmelCase			: Optional[int]     					=			0.1 if task == '''translation''' else 0

     _UpperCAmelCase			: Optional[Any]     					=			make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
     _UpperCAmelCase			: Tuple     					=			self.get_auto_remove_tmp_dir()
     args_d.update(
         data_dir=_A							,					model_name_or_path=_A							,					tokenizer_name=_A							,					train_batch_size=2							,					eval_batch_size=2							,					output_dir=_A							,					do_predict=_A							,					task=_A							,					src_lang='''en_XX'''							,					tgt_lang='''ro_RO'''							,					freeze_encoder=_A							,					freeze_embeds=_A							,					)
     assert "n_train" in args_d
     _UpperCAmelCase			: Union[str, Any]     					=			argparse.Namespace(**_A)
     _UpperCAmelCase			: Dict     					=			main(_A)

     _UpperCAmelCase			: Dict     					=			module.model.get_input_embeddings()
     assert not input_embeds.weight.requires_grad
     if model == T5_TINY:
         _UpperCAmelCase			: List[Any]     					=			module.model.lm_head
         assert not lm_head.weight.requires_grad
         assert (lm_head.weight == input_embeds.weight).all().item()
     elif model == FSMT_TINY:
         _UpperCAmelCase			: Optional[int]     					=			module.model.model
         _UpperCAmelCase			: Union[str, Any]     					=			fsmt.decoder.embed_positions
         assert not embed_pos.weight.requires_grad
         assert not fsmt.decoder.embed_tokens.weight.requires_grad
         # check that embeds are not the same
         assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
     else:
         _UpperCAmelCase			: Optional[Any]     					=			module.model.model
         _UpperCAmelCase			: Any     					=			bart.decoder.embed_positions
         assert not embed_pos.weight.requires_grad
         assert not bart.shared.weight.requires_grad
         # check that embeds are the same
         assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
         assert bart.decoder.embed_tokens == bart.shared

     _UpperCAmelCase			: Tuple     					=			load_json(module.output_dir / '''text_batch.json''')
     assert isinstance(_A							,					_A)
     assert len(_A) >= 4


 def 	snake_case__			(		self)	->						Optional[int]:

     """simple docstring"""



     _UpperCAmelCase			: dict     					=			CHEAP_ARGS.copy()

     _UpperCAmelCase			: Optional[Any]     					=			'''summarization'''
     _UpperCAmelCase			: Union[str, Any]     					=			make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())

     args_d.update(
         data_dir=_A							,					tokenizer_name=_A							,					train_batch_size=2							,					eval_batch_size=2							,					do_predict=_A							,					task=_A							,					src_lang='''en_XX'''							,					tgt_lang='''ro_RO'''							,					freeze_encoder=_A							,					freeze_embeds=_A							,					)

     # test models whose config includes the extra_model_args
     _UpperCAmelCase			: Dict     					=			BART_TINY
     _UpperCAmelCase			: int     					=			self.get_auto_remove_tmp_dir()
     _UpperCAmelCase			: Optional[Any]     					=			args_d.copy()
     args_da.update(
         model_name_or_path=_A							,					output_dir=_A							,					)
     _UpperCAmelCase			: Any     					=			('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
     for p in extra_model_params:
         _UpperCAmelCase			: List[str]     					=			0.5
     _UpperCAmelCase			: Optional[int]     					=			argparse.Namespace(**_A)
     _UpperCAmelCase			: str     					=			main(_A)
     for p in extra_model_params:
         assert getattr(model.config							,					_A) == 0.5, f'''failed to override the model config for param {p}'''

     # test models whose config doesn't include the extra_model_args
     _UpperCAmelCase			: Optional[int]     					=			T5_TINY
     _UpperCAmelCase			: Union[str, Any]     					=			self.get_auto_remove_tmp_dir()
     _UpperCAmelCase			: Tuple     					=			args_d.copy()
     args_da.update(
         model_name_or_path=_A							,					output_dir=_A							,					)
     _UpperCAmelCase			: Optional[int]     					=			'''encoder_layerdrop'''
     _UpperCAmelCase			: List[Any]     					=			0.5
     _UpperCAmelCase			: Tuple     					=			argparse.Namespace(**_A)
     with pytest.raises(_A) as excinfo:
         _UpperCAmelCase			: str     					=			main(_A)
     assert str(excinfo.value) == f'''model config doesn\'t have a `{unsupported_param}` attribute'''


 def 	snake_case__			(		self)	->						Dict:

     """simple docstring"""



     _UpperCAmelCase			: dict     					=			CHEAP_ARGS.copy()

     _UpperCAmelCase			: Tuple     					=			'''summarization'''
     _UpperCAmelCase			: Tuple     					=			make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())

     _UpperCAmelCase			: List[Any]     					=			BART_TINY
     _UpperCAmelCase			: Any     					=			self.get_auto_remove_tmp_dir()

     args_d.update(
         data_dir=_A							,					model_name_or_path=_A							,					output_dir=_A							,					tokenizer_name=_A							,					train_batch_size=2							,					eval_batch_size=2							,					do_predict=_A							,					task=_A							,					src_lang='''en_XX'''							,					tgt_lang='''ro_RO'''							,					freeze_encoder=_A							,					freeze_embeds=_A							,					)

     # emulate finetune.py
     _UpperCAmelCase			: Optional[Any]     					=			argparse.ArgumentParser()
     _UpperCAmelCase			: int     					=			pl.Trainer.add_argparse_args(_A)
     _UpperCAmelCase			: Tuple     					=			SummarizationModule.add_model_specific_args(_A							,					os.getcwd())
     _UpperCAmelCase			: str     					=			{'''--help''': True}

     # --help test
     with pytest.raises(_A) as excinfo:
         with CaptureStdout() as cs:
             _UpperCAmelCase			: Optional[int]     					=			parser.parse_args(_A)
         assert False, "--help is expected to sys.exit"
     assert excinfo.type == SystemExit
     _UpperCAmelCase			: Optional[int]     					=			lightning_base.arg_to_scheduler_metavar
     assert expected in cs.out, "--help is expected to list the supported schedulers"

     # --lr_scheduler=non_existing_scheduler test
     _UpperCAmelCase			: int     					=			'''non_existing_scheduler'''
     _UpperCAmelCase			: Dict     					=			{f'''--lr_scheduler={unsupported_param}'''}
     with pytest.raises(_A) as excinfo:
         with CaptureStderr() as cs:
             _UpperCAmelCase			: Any     					=			parser.parse_args(_A)
         assert False, "invalid argument is expected to sys.exit"
     assert excinfo.type == SystemExit
     _UpperCAmelCase			: int     					=			f'''invalid choice: \'{unsupported_param}\''''
     assert expected in cs.err, f'''should have bailed on invalid choice of scheduler {unsupported_param}'''

     # --lr_scheduler=existing_scheduler test
     _UpperCAmelCase			: Optional[int]     					=			'''cosine'''
     _UpperCAmelCase			: Dict     					=			args_d.copy()
     _UpperCAmelCase			: Optional[int]     					=			supported_param
     _UpperCAmelCase			: Optional[Any]     					=			argparse.Namespace(**_A)
     _UpperCAmelCase			: Optional[int]     					=			main(_A)
     assert (
         getattr(model.hparams							,					'''lr_scheduler''') == supported_param
     ), f'''lr_scheduler={supported_param} shouldn\'t fail'''












import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)
class 							A_      :






 '''simple docstring'''




 def __init__(		self							,					_A)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: List[Any]     					=			foo


 def __call__(		self)	->						List[Any]:

     """simple docstring"""



     return self.foo



class 							A_      (  datasets.Dataset     ):






 '''simple docstring'''




 @fingerprint_transform(inplace=_A)
 def 	snake_case__			(		self							,					_A							,					*_A							,					**_A)	->						str:

     """simple docstring"""



     return DatasetChild(self.data							,					fingerprint=_A)


 @fingerprint_transform(inplace=_A)
 def 	snake_case__			(		self							,					_A							,					*_A							,					**_A)	->						Optional[Any]:

     """simple docstring"""



     return DatasetChild(self.data							,					fingerprint=_A)



class 							A_      :






 '''simple docstring'''




 def __init__(		self							,					_A)	->						List[str]:

     """simple docstring"""



     _UpperCAmelCase			: Union[str, Any]     					=			callable


 def __call__(		self							,					*_A							,					**_A)	->						Optional[int]:

     """simple docstring"""



     if self.callable is not None:
         return self.callable(*_A							,					**_A)


 def __getstate__(		self)	->						Optional[Any]:

     """simple docstring"""



     raise pickle.PicklingError()



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 @require_transformers
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     from transformers import AutoTokenizer

     def encode(_A):
         return tokenizer(_A)

     # TODO: add hash consistency tests across sessions
     _UpperCAmelCase			: Tuple     					=			AutoTokenizer.from_pretrained('''bert-base-uncased''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''bert-base-cased''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''bert-base-uncased''')
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_transformers
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Any:

     """simple docstring"""



     from transformers import AutoTokenizer

     _UpperCAmelCase			: Union[str, Any]     					=			AutoTokenizer.from_pretrained('''gpt2''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     tokenizer('''Hello world !''')  # call once to change the tokenizer's cache
     _UpperCAmelCase			: Optional[int]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)


 @require_regex
 def 	snake_case__			(		self)	->						Dict:

     """simple docstring"""



     import regex

     _UpperCAmelCase			: Optional[int]     					=			regex.Regex('''foo''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: int     					=			regex.Regex('''bar''')
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			regex.Regex('''foo''')
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     def func():
         return foo

     _UpperCAmelCase			: List[str]     					=			[0]
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			[1]
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Any     					=			[0]
     _UpperCAmelCase			: List[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						List[Any]:

     """simple docstring"""



     def func():
         pass

     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()

     def func():
         pass

     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
     _UpperCAmelCase			: int     					=			mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: List[str]     					=			(
         '''co_argcount''',
         '''co_kwonlyargcount''',
         '''co_nlocals''',
         '''co_stacksize''',
         '''co_flags''',
         '''co_code''',
         '''co_consts''',
         '''co_names''',
         '''co_varnames''',
         '''co_filename''',
         '''co_name''',
         '''co_firstlineno''',
         '''co_lnotab''',
         '''co_freevars''',
         '''co_cellvars''',
     )

     def _create_code_pyaa(*_A):
         if hasattr(args[-3]							,					'''encode'''):
             _UpperCAmelCase			: Tuple     					=			list(_A)
             _UpperCAmelCase			: Optional[int]     					=			args[-3].encode()  # co_lnotab
             _UpperCAmelCase			: List[str]     					=			args[-10].encode()  # co_code
         if hasattr(_A							,					'''co_posonlyargcount'''):
             if len(_A) == 16:
                 return CodeType(*_A)
             elif len(_A) == 15:
                 return CodeType(args[0]							,					0							,					*args[1:])
             return CodeType(args[0]							,					0							,					0							,					*args[1:])
         elif hasattr(_A							,					'''co_kwonlyargcount'''):
             if len(_A) == 16:
                 return CodeType(args[0]							,					*args[2:])
             elif len(_A) == 15:
                 return CodeType(*_A)
             return CodeType(args[0]							,					0							,					*args[1:])
         if len(_A) == 16:
             return CodeType(args[0]							,					*args[3:])
         elif len(_A) == 15:
             return CodeType(args[0]							,					*args[2:])
         return CodeType(*_A)

     def create_ipython_func(_A							,					_A):
         def func():
             return returned_obj

         _UpperCAmelCase			: List[Any]     					=			func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse('''3.8'''):
             _UpperCAmelCase			: int     					=			_create_code_pyaa(
                 *[getattr(_A							,					_A) if k != '''co_filename''' else co_filename for k in code_args_pyaa])
         else:
             _UpperCAmelCase			: List[str]     					=			code.replace(co_filename=_A)
         return FunctionType(_A							,					func.__globals__							,					func.__name__							,					func.__defaults__							,					func.__closure__)

     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-2-e0383a102aae>''', [0]
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-2-e0383a102aae>''', [1]
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: List[Any]     					=			'''<ipython-input-5-713f6613acf3>''', [0]
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)

     _UpperCAmelCase					,	_UpperCAmelCase			: Union[str, Any]     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''321456789.py'''), [0]
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: str     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''321456789.py'''), [1]
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     _UpperCAmelCase					,	_UpperCAmelCase			: Union[str, Any]     					=			os.path.join(gettempdir()							,					'''ipykernel_12345'''							,					'''654123987.py'''), [0]
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A							,					_A))).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase					,	_UpperCAmelCase			: List[str]     					=			[0], [1]

     def func():
         return foo, bar

     _UpperCAmelCase			: List[str]     					=			'''__main__'''

     def globalvars_mocka_side_effect(_A							,					*_A							,					**_A):
         return {"foo": foo, "bar": bar}

     def globalvars_mocka_side_effect(_A							,					*_A							,					**_A):
         return {"bar": bar, "foo": foo}

     with patch('''dill.detect.globalvars'''							,					side_effect=_A) as globalvars_mocka:
         _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count							,					0)
     with patch('''dill.detect.globalvars'''							,					side_effect=_A) as globalvars_mocka:
         _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count							,					0)
     self.assertEqual(_A							,					_A)



class 							A_      (  __lowercase     ):






 '''simple docstring'''




 def 	snake_case__			(		self)	->						List[str]:

     """simple docstring"""



     _UpperCAmelCase			: Optional[int]     					=			Hasher.hash('''hello''')
     _UpperCAmelCase			: Union[str, Any]     					=			Hasher.hash('''hello''')
     _UpperCAmelCase			: Tuple     					=			Hasher.hash('''there''')
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase			: Union[str, Any]     					=			Hasher.hash(Foo('''hello'''))
     _UpperCAmelCase			: Optional[int]     					=			Hasher.hash(Foo('''hello'''))
     _UpperCAmelCase			: Any     					=			Hasher.hash(Foo('''there'''))
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Optional[int]:

     """simple docstring"""



     _UpperCAmelCase			: List[Any]     					=			Hasher()
     for x in ["hello", Foo('''hello''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[Any]     					=			hasher.hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			Hasher()
     for x in ["hello", Foo('''hello''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[int]     					=			hasher.hexdigest()
     _UpperCAmelCase			: Optional[int]     					=			Hasher()
     for x in ["there", Foo('''there''')]:
         hasher.update(_A)
     _UpperCAmelCase			: Optional[int]     					=			hasher.hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 def 	snake_case__			(		self)	->						Optional[Any]:

     """simple docstring"""



     with self.assertRaises(pickle.PicklingError):
         Hasher.hash(UnpicklableCallable(Foo('''hello''')))


 def 	snake_case__			(		self)	->						int:

     """simple docstring"""



     _UpperCAmelCase			: List[str]     					=			'''abc'''
     _UpperCAmelCase			: List[Any]     					=			[string, string]  # two strings have the same ids
     _UpperCAmelCase			: int     					=			[string, string]
     _UpperCAmelCase			: List[str]     					=			json.loads(f'''["{string}", "{string}"]''')  # two strings have different ids
     self.assertIs(obja[0]							,					_A)
     self.assertIs(obja[0]							,					obja[1])
     self.assertIs(obja[0]							,					_A)
     self.assertIs(obja[0]							,					obja[1])
     self.assertIsNot(obja[0]							,					_A)
     self.assertIsNot(obja[0]							,					obja[1])
     _UpperCAmelCase			: List[Any]     					=			Hasher.hash(_A)
     _UpperCAmelCase			: int     					=			Hasher.hash(_A)
     _UpperCAmelCase			: List[str]     					=			Hasher.hash(_A)
     self.assertEqual(_A							,					_A)
     self.assertEqual(_A							,					_A)


 @require_tiktoken
 def 	snake_case__			(		self)	->						Any:

     """simple docstring"""



     import tiktoken

     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''gpt2''')
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''r50k_base''')
     _UpperCAmelCase			: Optional[Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			tiktoken.get_encoding('''gpt2''')
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_torch
 def 	snake_case__			(		self)	->						Optional[int]:

     """simple docstring"""



     import torch

     _UpperCAmelCase			: Optional[int]     					=			torch.tensor([1.0])
     _UpperCAmelCase			: Any     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Optional[Any]     					=			torch.tensor([2.0])
     _UpperCAmelCase			: List[str]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Any     					=			torch.tensor([1.0])
     _UpperCAmelCase			: Dict     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)


 @require_spacy
 @require_spacy_model('''en_core_web_sm''')
 @require_spacy_model('''fr_core_news_sm''')
 @pytest.mark.integration
 def 	snake_case__			(		self)	->						Tuple:

     """simple docstring"""



     import spacy

     _UpperCAmelCase			: Optional[int]     					=			spacy.load('''en_core_web_sm''')
     _UpperCAmelCase			: Union[str, Any]     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: Dict     					=			spacy.load('''fr_core_news_sm''')
     _UpperCAmelCase			: str     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     _UpperCAmelCase			: List[str]     					=			spacy.load('''en_core_web_sm''')
     _UpperCAmelCase			: Tuple     					=			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
     self.assertEqual(_A							,					_A)
     self.assertNotEqual(_A							,					_A)



@pytest.mark.integration
def        _lowerCamelCase		(  __A      :							Path	) ->   int:
    _UpperCAmelCase			: Union[str, Any]     					=			tmp_path / '''dir1'''
    _UpperCAmelCase			: int     					=			tmp_path / '''dir2'''
    dira.mkdir()
    dira.mkdir()
    _UpperCAmelCase			: List[str]     					=			'''script.py'''
    _UpperCAmelCase			: str     					=			dedent(
        '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''	)
    _UpperCAmelCase			: Optional[Any]     					=			dira / script_filename
    _UpperCAmelCase			: List[Any]     					=			dira / script_filename
    with script_patha.open('''w'''	) as f:
        f.write(__A	)
    with script_patha.open('''w'''	) as f:
        f.write(__A	)
    _UpperCAmelCase			: Union[str, Any]     					=			subprocess.check_output(['''python''', str(__A	)]	)
    _UpperCAmelCase			: str     					=			subprocess.check_output(['''python''', str(__A	)]	)
    assert fingerprinta == fingerprinta

def        _lowerCamelCase		(  ) ->   Optional[int]:
    _UpperCAmelCase			: Union[str, Any]     					=			{'''a''': [0, 1, 2]}
    _UpperCAmelCase			: Union[str, Any]     					=			DatasetChild(InMemoryTable.from_pydict(__A	)	)
    _UpperCAmelCase			: Optional[Any]     					=			dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2	) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca	).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca	).get()._fingerprint

def        _lowerCamelCase		(  ) ->   str:
    _UpperCAmelCase			: str     					=			{'''a''': [0, 1, 2]}
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A)
     def 	snake_case__			(		self							,					_A)	->						Optional[int]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: Tuple     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A							,					version='''1.0.0''')
     def 	snake_case__			(		self							,					_A)	->						List[str]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: str     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()
    class 							A_      (  datasets.Dataset     ):






     '''simple docstring'''




     @fingerprint_transform(inplace=_A							,					version='''2.0.0''')
     def 	snake_case__			(		self							,					_A)	->						List[str]:

         """simple docstring"""



         return DummyDatasetChild(self.data							,					fingerprint=_A)



    _UpperCAmelCase			: str     					=			DummyDatasetChild(InMemoryTable.from_pydict(__A	)	).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}	) == 3


def        _lowerCamelCase		(  ) ->   List[str]:
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    _UpperCAmelCase			: Dict     					=			Hasher()
    hasher.update(lambda __A	: x	)




