

import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class    a__  :
			"""simple docstring"""







			def __init__(      self , lowercase    )					->  List[Any]:
							'''simple docstring'''






							A__            =     foo


			def __call__(      self    )					->  Optional[int]:
							'''simple docstring'''






							return self.foo

class    a__  (   datasets.Dataset     ):
			"""simple docstring"""







			@fingerprint_transform(inplace=lowercase    )
			def        UpperCamelCase						(      self , lowercase , *lowercase , **lowercase    )					->  Tuple:
							'''simple docstring'''






							return DatasetChild(self.data , fingerprint=lowercase    )


			@fingerprint_transform(inplace=lowercase    )
			def        UpperCamelCase						(      self , lowercase , *lowercase , **lowercase    )					->  Tuple:
							'''simple docstring'''






							return DatasetChild(self.data , fingerprint=lowercase    )

class    a__  :
			"""simple docstring"""







			def __init__(      self , lowercase    )					->  Optional[int]:
							'''simple docstring'''






							A__            =     callable


			def __call__(      self , *lowercase , **lowercase    )					->  List[Any]:
							'''simple docstring'''






							if self.callable is not None:
											return self.callable(*lowercase , **lowercase    )


			def __getstate__(      self    )					->  Any:
							'''simple docstring'''






							raise pickle.PicklingError()

class    a__  (   snake_case     ):
			"""simple docstring"""







			@require_transformers
			@pytest.mark.integration
			def        UpperCamelCase						(      self    )					->  str:
							'''simple docstring'''






							from transformers import AutoTokenizer

							def encode(lowercase    ):
											return tokenizer(lowercase    )

							# TODO: add hash consistency tests across sessions
							A__            =     AutoTokenizer.from_pretrained("bert-base-uncased"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lambda lowercase    : tokenizer(lowercase    )    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     AutoTokenizer.from_pretrained("bert-base-cased"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lambda lowercase    : tokenizer(lowercase    )    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     AutoTokenizer.from_pretrained("bert-base-uncased"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lambda lowercase    : tokenizer(lowercase    )    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			@require_transformers
			@pytest.mark.integration
			def        UpperCamelCase						(      self    )					->  Dict:
							'''simple docstring'''






							from transformers import AutoTokenizer

							A__            =     AutoTokenizer.from_pretrained("gpt2"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							tokenizer("Hello world !"    )  # call once to change the tokenizer's cache
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )


			@require_regex
			def        UpperCamelCase						(      self    )					->  Union[str, Any]:
							'''simple docstring'''






							import regex

							A__            =     regex.Regex("foo"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     regex.Regex("bar"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     regex.Regex("foo"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )

class    a__  (   snake_case     ):
			"""simple docstring"""







			def        UpperCamelCase						(      self    )					->  int:
							'''simple docstring'''






							def func():
											return foo

							A__            =     [0]
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     [1]
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     [0]
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  Optional[Any]:
							'''simple docstring'''






							def func():
											pass

							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()

							def func():
											pass

							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  List[Any]:
							'''simple docstring'''






							A__            =     mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(Foo([1]    )    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  Optional[Any]:
							'''simple docstring'''






							A__            =     mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(Foo([1]    ).__call__    )    ).hexdigest()
							A__            =     mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  str:
							'''simple docstring'''






							A__            =     (
							    "co_argcount",
							    "co_kwonlyargcount",
							    "co_nlocals",
							    "co_stacksize",
							    "co_flags",
							    "co_code",
							    "co_consts",
							    "co_names",
							    "co_varnames",
							    "co_filename",
							    "co_name",
							    "co_firstlineno",
							    "co_lnotab",
							    "co_freevars",
							    "co_cellvars",
							)

							def _create_code_pyaa(*lowercase    ):
											if hasattr(args[-3] , "encode"    ):
															A__            =     list(lowercase    )
															A__            =     args[-3].encode()  # co_lnotab
															A__            =     args[-10].encode()  # co_code
											if hasattr(lowercase , "co_posonlyargcount"    ):
															if len(lowercase    ) == 16:
																			return CodeType(*lowercase    )
															elif len(lowercase    ) == 15:
																			return CodeType(args[0] , 0 , *args[1:]    )
															return CodeType(args[0] , 0 , 0 , *args[1:]    )
											elif hasattr(lowercase , "co_kwonlyargcount"    ):
															if len(lowercase    ) == 16:
																			return CodeType(args[0] , *args[2:]    )
															elif len(lowercase    ) == 15:
																			return CodeType(*lowercase    )
															return CodeType(args[0] , 0 , *args[1:]    )
											if len(lowercase    ) == 16:
															return CodeType(args[0] , *args[3:]    )
											elif len(lowercase    ) == 15:
															return CodeType(args[0] , *args[2:]    )
											return CodeType(*lowercase    )

							def create_ipython_func(lowercase , lowercase    ):
											def func():
															return returned_obj

											A__            =     func.__code__
											# Use _create_code from dill in order to make it work for different python versions
											if datasets.config.PY_VERSION < version.parse("3.8"    ):
															A__            =     _create_code_pyaa(
															    *[getattr(lowercase , lowercase    ) if k != "co_filename" else co_filename for k in code_args_pyaa]    )
											else:
															A__            =     code.replace(co_filename=lowercase    )
											return FunctionType(lowercase , func.__globals__ , func.__name__ , func.__defaults__ , func.__closure__    )

							A__      ,			A__            =     "<ipython-input-2-e0383a102aae>", [0]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							A__      ,			A__            =     "<ipython-input-2-e0383a102aae>", [1]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							A__      ,			A__            =     "<ipython-input-5-713f6613acf3>", [0]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )

							A__      ,			A__            =     os.path.join(gettempdir() , "ipykernel_12345" , "321456789.py"    ), [0]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							A__      ,			A__            =     os.path.join(gettempdir() , "ipykernel_12345" , "321456789.py"    ), [1]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							A__      ,			A__            =     os.path.join(gettempdir() , "ipykernel_12345" , "654123987.py"    ), [0]
							A__            =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase , lowercase    )    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  Tuple:
							'''simple docstring'''






							A__      ,			A__            =     [0], [1]

							def func():
											return foo, bar

							A__            =     "__main__"

							def globalvars_mocka_side_effect(lowercase , *lowercase , **lowercase    ):
											return {"foo": foo, "bar": bar}

							def globalvars_mocka_side_effect(lowercase , *lowercase , **lowercase    ):
											return {"bar": bar, "foo": foo}

							with patch("dill.detect.globalvars" , side_effect=lowercase    ) as globalvars_mocka:
											A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
											self.assertGreater(globalvars_mocka.call_count , 0    )
							with patch("dill.detect.globalvars" , side_effect=lowercase    ) as globalvars_mocka:
											A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
											self.assertGreater(globalvars_mocka.call_count , 0    )
							self.assertEqual(lowercase , lowercase    )

class    a__  (   snake_case     ):
			"""simple docstring"""







			def        UpperCamelCase						(      self    )					->  List[str]:
							'''simple docstring'''






							A__            =     Hasher.hash("hello"    )
							A__            =     Hasher.hash("hello"    )
							A__            =     Hasher.hash("there"    )
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  int:
							'''simple docstring'''






							A__            =     Hasher.hash(Foo("hello"    )    )
							A__            =     Hasher.hash(Foo("hello"    )    )
							A__            =     Hasher.hash(Foo("there"    )    )
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  str:
							'''simple docstring'''






							A__            =     Hasher()
							for x in ["hello", Foo("hello"    )]:
											hasher.update(lowercase    )
							A__            =     hasher.hexdigest()
							A__            =     Hasher()
							for x in ["hello", Foo("hello"    )]:
											hasher.update(lowercase    )
							A__            =     hasher.hexdigest()
							A__            =     Hasher()
							for x in ["there", Foo("there"    )]:
											hasher.update(lowercase    )
							A__            =     hasher.hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			def        UpperCamelCase						(      self    )					->  int:
							'''simple docstring'''






							with self.assertRaises(pickle.PicklingError    ):
											Hasher.hash(UnpicklableCallable(Foo("hello"    )    )    )


			def        UpperCamelCase						(      self    )					->  Dict:
							'''simple docstring'''






							A__            =     "abc"
							A__            =     [string, string]  # two strings have the same ids
							A__            =     [string, string]
							A__            =     json.loads(F'["{string}", "{string}"]'    )  # two strings have different ids
							self.assertIs(obja[0] , lowercase    )
							self.assertIs(obja[0] , obja[1]    )
							self.assertIs(obja[0] , lowercase    )
							self.assertIs(obja[0] , obja[1]    )
							self.assertIsNot(obja[0] , lowercase    )
							self.assertIsNot(obja[0] , obja[1]    )
							A__            =     Hasher.hash(lowercase    )
							A__            =     Hasher.hash(lowercase    )
							A__            =     Hasher.hash(lowercase    )
							self.assertEqual(lowercase , lowercase    )
							self.assertEqual(lowercase , lowercase    )


			@require_tiktoken
			def        UpperCamelCase						(      self    )					->  List[str]:
							'''simple docstring'''






							import tiktoken

							A__            =     tiktoken.get_encoding("gpt2"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     tiktoken.get_encoding("r50k_base"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     tiktoken.get_encoding("gpt2"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			@require_torch
			def        UpperCamelCase						(      self    )					->  Optional[int]:
							'''simple docstring'''






							import torch

							A__            =     torch.tensor([1.0]    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     torch.tensor([2.0]    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     torch.tensor([1.0]    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )


			@require_spacy
			@require_spacy_model("en_core_web_sm"    )
			@require_spacy_model("fr_core_news_sm"    )
			@pytest.mark.integration
			def        UpperCamelCase						(      self    )					->  Optional[Any]:
							'''simple docstring'''






							import spacy

							A__            =     spacy.load("en_core_web_sm"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     spacy.load("fr_core_news_sm"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							A__            =     spacy.load("en_core_web_sm"    )
							A__            =     mda(datasets.utils.py_utils.dumps(lowercase    )    ).hexdigest()
							self.assertEqual(lowercase , lowercase    )
							self.assertNotEqual(lowercase , lowercase    )



@pytest.mark.integration
def    lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Path )     -> Optional[int]:




				'''simple docstring'''





				A__            =     tmp_path / "dir1"
				A__            =     tmp_path / "dir2"
				dira.mkdir()
				dira.mkdir()
				A__            =     "script.py"
				A__            =     dedent(
				    "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    " )
				A__            =     dira / script_filename
				A__            =     dira / script_filename
				with script_patha.open("w" ) as f:
								f.write(SCREAMING_SNAKE_CASE_ )
				with script_patha.open("w" ) as f:
								f.write(SCREAMING_SNAKE_CASE_ )
				A__            =     subprocess.check_output(["python", str(SCREAMING_SNAKE_CASE_ )] )
				A__            =     subprocess.check_output(["python", str(SCREAMING_SNAKE_CASE_ )] )
				assert fingerprinta == fingerprinta



def    lowerCAmelCase__							(		)     -> Any:




				'''simple docstring'''





				A__            =     {"a": [0, 1, 2]}
				A__            =     DatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_ ) )
				A__            =     dataset.funca()._fingerprint
				assert expected_fingerprint == dataset.funca()._fingerprint
				assert expected_fingerprint != dataset.funca()._fingerprint

				with Pool(2 ) as p:
								assert expected_fingerprint == p.apply_async(dataset.funca ).get()._fingerprint
								assert expected_fingerprint != p.apply_async(dataset.funca ).get()._fingerprint



def    lowerCAmelCase__							(		)     -> int:




				'''simple docstring'''





				A__            =     {"a": [0, 1, 2]}

				class    a__  (   datasets.Dataset     ):
							"""simple docstring"""







							@fingerprint_transform(inplace=lowercase    )
							def        UpperCamelCase						(      self , lowercase    )					->  int:
											'''simple docstring'''






											return DummyDatasetChild(self.data , fingerprint=lowercase    )

				A__            =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_ ) ).func()

				class    a__  (   datasets.Dataset     ):
							"""simple docstring"""







							@fingerprint_transform(inplace=lowercase , version="1.0.0"    )
							def        UpperCamelCase						(      self , lowercase    )					->  List[str]:
											'''simple docstring'''






											return DummyDatasetChild(self.data , fingerprint=lowercase    )

				A__            =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_ ) ).func()

				class    a__  (   datasets.Dataset     ):
							"""simple docstring"""







							@fingerprint_transform(inplace=lowercase , version="2.0.0"    )
							def        UpperCamelCase						(      self , lowercase    )					->  List[Any]:
											'''simple docstring'''






											return DummyDatasetChild(self.data , fingerprint=lowercase    )

				A__            =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_ ) ).func()

				assert len({fingeprint_no_version, fingeprint_a, fingeprint_a} ) == 3





def    lowerCAmelCase__							(		)     -> Optional[Any]:




				'''simple docstring'''





				A__            =     Hasher()
				hasher.update(lambda SCREAMING_SNAKE_CASE_ : x )




from typing import Optional

from ..utils.logging import get_logger
from .audio_classificiation import AudioClassification
from .automatic_speech_recognition import AutomaticSpeechRecognition
from .base import TaskTemplate
from .image_classification import ImageClassification
from .language_modeling import LanguageModeling
from .question_answering import QuestionAnsweringExtractive
from .summarization import Summarization
from .text_classification import TextClassification


lowerCAmelCase__						=	[
    """AutomaticSpeechRecognition""",
    """AudioClassification""",
    """ImageClassification""",
    """LanguageModeling""",
    """QuestionAnsweringExtractive""",
    """Summarization""",
    """TaskTemplate""",
    """TextClassification""",
]

lowerCAmelCase__						=	get_logger(__name__)


lowerCAmelCase__						=	{
    AutomaticSpeechRecognition.task: AutomaticSpeechRecognition,
    AudioClassification.task: AudioClassification,
    ImageClassification.task: ImageClassification,
    LanguageModeling.task: LanguageModeling,
    QuestionAnsweringExtractive.task: QuestionAnsweringExtractive,
    Summarization.task: Summarization,
    TextClassification.task: TextClassification,
}



def    lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							dict )     -> Optional[TaskTemplate]:




				'''simple docstring'''





				A__            =     task_template_dict.get("task" )
				if task_name is None:
								logger.warning(F'Couldn\'t find template for task \'{task_name}\'. Available templates: {list(SCREAMING_SNAKE_CASE_ )}' )
								return None
				A__            =     NAMEaTEMPLATE.get(SCREAMING_SNAKE_CASE_ )
				return template.from_dict(SCREAMING_SNAKE_CASE_ )

