
"""simple docstring"""




import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class 					UpperCamelCase     :

    """simple docstring"""



    def __init__(    self		,     lowercase__      )   ->   List[str]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     foo




    def __call__(    self      )   ->   int:
         """simple docstring"""







         return self.foo






class 					UpperCamelCase     (	datasets.Dataset				):

    """simple docstring"""



    @fingerprint_transform(inplace=lowercase__      )
    def 							A (    self		,     lowercase__		,     *lowercase__		,     **lowercase__      )   ->   Tuple:
         """simple docstring"""







         return DatasetChild(self.data		,     fingerprint=lowercase__      )




    @fingerprint_transform(inplace=lowercase__      )
    def 							A (    self		,     lowercase__		,     *lowercase__		,     **lowercase__      )   ->   List[Any]:
         """simple docstring"""







         return DatasetChild(self.data		,     fingerprint=lowercase__      )






class 					UpperCamelCase     :

    """simple docstring"""



    def __init__(    self		,     lowercase__      )   ->   str:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     callable


    def __call__(    self		,     *lowercase__		,     **lowercase__      )   ->   List[str]:
         """simple docstring"""







         if self.callable is not None:
              return self.callable(*lowercase__		,     **lowercase__      )




    def __getstate__(    self      )   ->   Optional[Any]:
         """simple docstring"""







         raise pickle.PicklingError()






class 					UpperCamelCase     (	__magic_name__				):

    """simple docstring"""



    @require_transformers
    @pytest.mark.integration
    def 							A (    self      )   ->   str:
         """simple docstring"""







         from transformers import AutoTokenizer

         def encode(lowercase__      ):
              return tokenizer(lowercase__      )

         # TODO: add hash consistency tests across sessions
         SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained('bert-base-uncased'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lambda lowercase__      : tokenizer(lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained('bert-base-cased'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lambda lowercase__      : tokenizer(lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained('bert-base-uncased'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lambda lowercase__      : tokenizer(lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    @require_transformers
    @pytest.mark.integration
    def 							A (    self      )   ->   Any:
         """simple docstring"""







         from transformers import AutoTokenizer

         SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained('gpt2'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         tokenizer('Hello world !'      )  # call once to change the tokenizer's cache
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )




    @require_regex
    def 							A (    self      )   ->   Tuple:
         """simple docstring"""







         import regex

         SCREAMING_SNAKE_CASE           =     regex.Regex('foo'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     regex.Regex('bar'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     regex.Regex('foo'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )






class 					UpperCamelCase     (	__magic_name__				):

    """simple docstring"""



    def 							A (    self      )   ->   Any:
         """simple docstring"""







         def func():
              return foo

         SCREAMING_SNAKE_CASE           =     [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     [1]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   Union[str, Any]:
         """simple docstring"""







         def func():
              pass

         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()

         def func():
              pass

         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   List[Any]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([0]      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([1]      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([0]      )      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([0]      ).__call__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([1]      ).__call__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(Foo([0]      ).__call__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   int:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     (
             'co_argcount',
             'co_kwonlyargcount',
             'co_nlocals',
             'co_stacksize',
             'co_flags',
             'co_code',
             'co_consts',
             'co_names',
             'co_varnames',
             'co_filename',
             'co_name',
             'co_firstlineno',
             'co_lnotab',
             'co_freevars',
             'co_cellvars',
         )

         def _create_code_pyaa(*lowercase__      ):
              if hasattr(args[-3]		,     'encode'      ):
                   SCREAMING_SNAKE_CASE           =     list(lowercase__      )
                   SCREAMING_SNAKE_CASE           =     args[-3].encode()  # co_lnotab
                   SCREAMING_SNAKE_CASE           =     args[-10].encode()  # co_code
              if hasattr(lowercase__		,     'co_posonlyargcount'      ):
                   if len(lowercase__      ) == 16:
                        return CodeType(*lowercase__      )
                   elif len(lowercase__      ) == 15:
                        return CodeType(args[0]		,     0		,     *args[1:]      )
                   return CodeType(args[0]		,     0		,     0		,     *args[1:]      )
              elif hasattr(lowercase__		,     'co_kwonlyargcount'      ):
                   if len(lowercase__      ) == 16:
                        return CodeType(args[0]		,     *args[2:]      )
                   elif len(lowercase__      ) == 15:
                        return CodeType(*lowercase__      )
                   return CodeType(args[0]		,     0		,     *args[1:]      )
              if len(lowercase__      ) == 16:
                   return CodeType(args[0]		,     *args[3:]      )
              elif len(lowercase__      ) == 15:
                   return CodeType(args[0]		,     *args[2:]      )
              return CodeType(*lowercase__      )

         def create_ipython_func(lowercase__		,     lowercase__      ):
              def func():
                   return returned_obj

              SCREAMING_SNAKE_CASE           =     func.__code__
              # Use _create_code from dill in order to make it work for different python versions
              if datasets.config.PY_VERSION < version.parse('3.8'      ):
                   SCREAMING_SNAKE_CASE           =     _create_code_pyaa(
                       *[getattr(lowercase__		,     lowercase__      ) if k != 'co_filename' else co_filename for k in code_args_pyaa]      )
              else:
                   SCREAMING_SNAKE_CASE           =     code.replace(co_filename=lowercase__      )
              return FunctionType(lowercase__		,     func.__globals__		,     func.__name__		,     func.__defaults__		,     func.__closure__      )

         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     '<ipython-input-2-e0383a102aae>', [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     '<ipython-input-2-e0383a102aae>', [1]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     '<ipython-input-5-713f6613acf3>', [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )

         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     os.path.join(gettempdir()		,     'ipykernel_12345'		,     '321456789.py'      ), [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     os.path.join(gettempdir()		,     'ipykernel_12345'		,     '321456789.py'      ), [1]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     os.path.join(gettempdir()		,     'ipykernel_12345'		,     '654123987.py'      ), [0]
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase__		,     lowercase__      )      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )




    def 							A (    self      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     [0], [1]

         def func():
              return foo, bar

         SCREAMING_SNAKE_CASE           =     '__main__'

         def globalvars_mocka_side_effect(lowercase__		,     *lowercase__		,     **lowercase__      ):
              return {"foo": foo, "bar": bar}

         def globalvars_mocka_side_effect(lowercase__		,     *lowercase__		,     **lowercase__      ):
              return {"bar": bar, "foo": foo}

         with patch('dill.detect.globalvars'		,     side_effect=lowercase__      ) as globalvars_mocka:
              SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
              self.assertGreater(globalvars_mocka.call_count		,     0      )
         with patch('dill.detect.globalvars'		,     side_effect=lowercase__      ) as globalvars_mocka:
              SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
              self.assertGreater(globalvars_mocka.call_count		,     0      )
         self.assertEqual(lowercase__		,     lowercase__      )






class 					UpperCamelCase     (	__magic_name__				):

    """simple docstring"""



    def 							A (    self      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     Hasher.hash('hello'      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash('hello'      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash('there'      )
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     Hasher.hash(Foo('hello'      )      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash(Foo('hello'      )      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash(Foo('there'      )      )
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   Any:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     Hasher()
         for x in ["hello", Foo('hello'      )]:
              hasher.update(lowercase__      )
         SCREAMING_SNAKE_CASE           =     hasher.hexdigest()
         SCREAMING_SNAKE_CASE           =     Hasher()
         for x in ["hello", Foo('hello'      )]:
              hasher.update(lowercase__      )
         SCREAMING_SNAKE_CASE           =     hasher.hexdigest()
         SCREAMING_SNAKE_CASE           =     Hasher()
         for x in ["there", Foo('there'      )]:
              hasher.update(lowercase__      )
         SCREAMING_SNAKE_CASE           =     hasher.hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    def 							A (    self      )   ->   int:
         """simple docstring"""







         with self.assertRaises(pickle.PicklingError      ):
              Hasher.hash(UnpicklableCallable(Foo('hello'      )      )      )


    def 							A (    self      )   ->   Optional[int]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     'abc'
         SCREAMING_SNAKE_CASE           =     [string, string]  # two strings have the same ids
         SCREAMING_SNAKE_CASE           =     [string, string]
         SCREAMING_SNAKE_CASE           =     json.loads(f'''["{string}", "{string}"]'''      )  # two strings have different ids
         self.assertIs(obja[0]		,     lowercase__      )
         self.assertIs(obja[0]		,     obja[1]      )
         self.assertIs(obja[0]		,     lowercase__      )
         self.assertIs(obja[0]		,     obja[1]      )
         self.assertIsNot(obja[0]		,     lowercase__      )
         self.assertIsNot(obja[0]		,     obja[1]      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash(lowercase__      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash(lowercase__      )
         SCREAMING_SNAKE_CASE           =     Hasher.hash(lowercase__      )
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertEqual(lowercase__		,     lowercase__      )


    @require_tiktoken
    def 							A (    self      )   ->   List[Any]:
         """simple docstring"""







         import tiktoken

         SCREAMING_SNAKE_CASE           =     tiktoken.get_encoding('gpt2'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     tiktoken.get_encoding('r50k_base'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     tiktoken.get_encoding('gpt2'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )


    @require_torch
    def 							A (    self      )   ->   str:
         """simple docstring"""







         import torch

         SCREAMING_SNAKE_CASE           =     torch.tensor([1.0]      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     torch.tensor([2.0]      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     torch.tensor([1.0]      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )




    @require_spacy
    @require_spacy_model('en_core_web_sm'      )
    @require_spacy_model('fr_core_news_sm'      )
    @pytest.mark.integration
    def 							A (    self      )   ->   Optional[Any]:
         """simple docstring"""







         import spacy

         SCREAMING_SNAKE_CASE           =     spacy.load('en_core_web_sm'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     spacy.load('fr_core_news_sm'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         SCREAMING_SNAKE_CASE           =     spacy.load('en_core_web_sm'      )
         SCREAMING_SNAKE_CASE           =     mda(datasets.utils.py_utils.dumps(lowercase__      )      ).hexdigest()
         self.assertEqual(lowercase__		,     lowercase__      )
         self.assertNotEqual(lowercase__		,     lowercase__      )






@pytest.mark.integration
def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
     SCREAMING_SNAKE_CASE           =     tmp_path / 'dir1'
     SCREAMING_SNAKE_CASE           =     tmp_path / 'dir2'
     dira.mkdir()
     dira.mkdir()
     SCREAMING_SNAKE_CASE           =     'script.py'
     SCREAMING_SNAKE_CASE           =     dedent(
         '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '      )
     SCREAMING_SNAKE_CASE           =     dira / script_filename
     SCREAMING_SNAKE_CASE           =     dira / script_filename
     with script_patha.open('w'      ) as f:
          f.write(SCREAMING_SNAKE_CASE_      )
     with script_patha.open('w'      ) as f:
          f.write(SCREAMING_SNAKE_CASE_      )
     SCREAMING_SNAKE_CASE           =     subprocess.check_output(['python', str(SCREAMING_SNAKE_CASE_      )]      )
     SCREAMING_SNAKE_CASE           =     subprocess.check_output(['python', str(SCREAMING_SNAKE_CASE_      )]      )
     assert fingerprinta == fingerprinta

def   UpperCamelCase_  (		):
     SCREAMING_SNAKE_CASE           =     {'a': [0, 1, 2]}
     SCREAMING_SNAKE_CASE           =     DatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_      )      )
     SCREAMING_SNAKE_CASE           =     dataset.funca()._fingerprint
     assert expected_fingerprint == dataset.funca()._fingerprint
     assert expected_fingerprint != dataset.funca()._fingerprint

     with Pool(2      ) as p:
          assert expected_fingerprint == p.apply_async(dataset.funca      ).get()._fingerprint
          assert expected_fingerprint != p.apply_async(dataset.funca      ).get()._fingerprint

def   UpperCamelCase_  (		):
     SCREAMING_SNAKE_CASE           =     {'a': [0, 1, 2]}




     class 					UpperCamelCase     (	datasets.Dataset				):

         """simple docstring"""



         @fingerprint_transform(inplace=lowercase__      )
         def 							A (    self		,     lowercase__      )   ->   Any:
              """simple docstring"""







              return DummyDatasetChild(self.data		,     fingerprint=lowercase__      )






     SCREAMING_SNAKE_CASE           =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_      )      ).func()




     class 					UpperCamelCase     (	datasets.Dataset				):

         """simple docstring"""



         @fingerprint_transform(inplace=lowercase__		,     version='1.0.0'      )
         def 							A (    self		,     lowercase__      )   ->   int:
              """simple docstring"""







              return DummyDatasetChild(self.data		,     fingerprint=lowercase__      )






     SCREAMING_SNAKE_CASE           =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_      )      ).func()




     class 					UpperCamelCase     (	datasets.Dataset				):

         """simple docstring"""



         @fingerprint_transform(inplace=lowercase__		,     version='2.0.0'      )
         def 							A (    self		,     lowercase__      )   ->   Dict:
              """simple docstring"""







              return DummyDatasetChild(self.data		,     fingerprint=lowercase__      )






     SCREAMING_SNAKE_CASE           =     DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE_      )      ).func()

     assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}      ) == 3



def   UpperCamelCase_  (		):
     # AttributeError: module 'dill._dill' has no attribute 'stack'
     SCREAMING_SNAKE_CASE           =     Hasher()
     hasher.update(lambda SCREAMING_SNAKE_CASE_      : x      )


"""simple docstring"""




import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


snake_case			       =		logging.get_logger(__name__)




@dataclass
class 					UpperCamelCase     :

				"""simple docstring"""



				UpperCAmelCase_    :				str
				UpperCAmelCase_    :				str
				UpperCAmelCase_    :				Optional[str]						  =    None
				UpperCAmelCase_    :				Optional[str]						  =    None


				def 							A (    self      )   ->   int:
									"""simple docstring"""







									return json.dumps(dataclasses.asdict(self      )		,     indent=2      ) + "\n"






@dataclass(frozen=__magic_name__				)
class 					UpperCamelCase     :

				"""simple docstring"""



				UpperCAmelCase_    :				List[int]
				UpperCAmelCase_    :				Optional[List[int]]						  =    None
				UpperCAmelCase_    :				Optional[List[int]]						  =    None
				UpperCAmelCase_    :				Optional[Union[int, float]]						  =    None


				def 							A (    self      )   ->   Union[str, Any]:
									"""simple docstring"""







									return json.dumps(dataclasses.asdict(self      )      ) + "\n"






class 					UpperCamelCase     :

				"""simple docstring"""



				def 							A (    self		,     lowercase__      )   ->   Union[str, Any]:
									"""simple docstring"""







									raise NotImplementedError()


				def 							A (    self		,     lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									raise NotImplementedError()


				def 							A (    self		,     lowercase__      )   ->   int:
									"""simple docstring"""







									raise NotImplementedError()


				def 							A (    self		,     lowercase__      )   ->   str:
									"""simple docstring"""







									raise NotImplementedError()


				def 							A (    self      )   ->   Tuple:
									"""simple docstring"""







									raise NotImplementedError()


				def 							A (    self		,     lowercase__      )   ->   str:
									"""simple docstring"""







									if len(self.get_labels()      ) > 1:
														SCREAMING_SNAKE_CASE           =     self.get_labels()[int(example.label      )]
									return example




				@classmethod
				def 							A (    cls		,     lowercase__		,     lowercase__=None      )   ->   Dict:
									"""simple docstring"""







									with open(lowercase__		,     'r'		,     encoding='utf-8-sig'      ) as f:
														return list(csv.reader(lowercase__		,     delimiter='\t'		,     quotechar=lowercase__      )      )






class 					UpperCamelCase     (	__magic_name__				):

				"""simple docstring"""



				def __init__(    self		,     lowercase__=None		,     lowercase__=None		,     lowercase__="classification"		,     lowercase__=False      )   ->   List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     [] if labels is None else labels
									SCREAMING_SNAKE_CASE           =     [] if examples is None else examples
									SCREAMING_SNAKE_CASE           =     mode
									SCREAMING_SNAKE_CASE           =     verbose


				def __len__(    self      )   ->   Optional[int]:
									"""simple docstring"""







									return len(self.examples      )


				def __getitem__(    self		,     lowercase__      )   ->   List[Any]:
									"""simple docstring"""







									if isinstance(lowercase__		,     lowercase__      ):
														return SingleSentenceClassificationProcessor(labels=self.labels		,     examples=self.examples[idx]      )
									return self.examples[idx]


				@classmethod
				def 							A (    cls		,     lowercase__		,     lowercase__=""		,     lowercase__=0		,     lowercase__=1		,     lowercase__=None		,     lowercase__=False		,     **lowercase__      )   ->   Tuple:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     cls(**lowercase__      )
									processor.add_examples_from_csv(
									    lowercase__		,     split_name=lowercase__		,     column_label=lowercase__		,     column_text=lowercase__		,     column_id=lowercase__		,     skip_first_row=lowercase__		,     overwrite_labels=lowercase__		,     overwrite_examples=lowercase__		,     )
									return processor


				@classmethod
				def 							A (    cls		,     lowercase__		,     lowercase__=None		,     **lowercase__      )   ->   int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     cls(**lowercase__      )
									processor.add_examples(lowercase__		,     labels=lowercase__      )
									return processor


				def 							A (    self		,     lowercase__		,     lowercase__=""		,     lowercase__=0		,     lowercase__=1		,     lowercase__=None		,     lowercase__=False		,     lowercase__=False		,     lowercase__=False		,     )   ->   List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self._read_tsv(lowercase__      )
									if skip_first_row:
														SCREAMING_SNAKE_CASE           =     lines[1:]
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									for i, line in enumerate(lowercase__      ):
														texts.append(line[column_text]      )
														labels.append(line[column_label]      )
														if column_id is not None:
																			ids.append(line[column_id]      )
														else:
																			SCREAMING_SNAKE_CASE           =     f'''{split_name}-{i}''' if split_name else str(lowercase__      )
																			ids.append(lowercase__      )

									return self.add_examples(
									    lowercase__		,     lowercase__		,     lowercase__		,     overwrite_labels=lowercase__		,     overwrite_examples=lowercase__      )


				def 							A (    self		,     lowercase__		,     lowercase__=None		,     lowercase__=None		,     lowercase__=False		,     lowercase__=False      )   ->   str:
									"""simple docstring"""







									if labels is not None and len(lowercase__      ) != len(lowercase__      ):
														raise ValueError(
														    f'''Text and labels have mismatched lengths {len(lowercase__      )} and {len(lowercase__      )}'''      )
									if ids is not None and len(lowercase__      ) != len(lowercase__      ):
														raise ValueError(f'''Text and ids have mismatched lengths {len(lowercase__      )} and {len(lowercase__      )}'''      )
									if ids is None:
														SCREAMING_SNAKE_CASE           =     [None] * len(lowercase__      )
									if labels is None:
														SCREAMING_SNAKE_CASE           =     [None] * len(lowercase__      )
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     set()
									for text_or_text_and_label, label, guid in zip(lowercase__		,     lowercase__		,     lowercase__      ):
														if isinstance(lowercase__		,     (tuple, list)      ) and label is None:
																			SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     text_or_text_and_label
														else:
																			SCREAMING_SNAKE_CASE           =     text_or_text_and_label
														added_labels.add(lowercase__      )
														examples.append(InputExample(guid=lowercase__		,     text_a=lowercase__		,     text_b=lowercase__		,     label=lowercase__      )      )

									# Update examples
									if overwrite_examples:
														SCREAMING_SNAKE_CASE           =     examples
									else:
														self.examples.extend(lowercase__      )

									# Update labels
									if overwrite_labels:
														SCREAMING_SNAKE_CASE           =     list(lowercase__      )
									else:
														SCREAMING_SNAKE_CASE           =     list(set(self.labels      ).union(lowercase__      )      )

									return self.examples




				def 							A (    self		,     lowercase__		,     lowercase__=None		,     lowercase__=False		,     lowercase__=0		,     lowercase__=True		,     lowercase__=None		,     )   ->   Optional[int]:
									"""simple docstring"""







									if max_length is None:
														SCREAMING_SNAKE_CASE           =     tokenizer.max_len

									SCREAMING_SNAKE_CASE           =     {label: i for i, label in enumerate(self.labels      )}

									SCREAMING_SNAKE_CASE           =     []
									for ex_index, example in enumerate(self.examples      ):
														if ex_index % 10000 == 0:
																			logger.info(f'''Tokenizing example {ex_index}'''      )

														SCREAMING_SNAKE_CASE           =     tokenizer.encode(
														    example.text_a		,     add_special_tokens=lowercase__		,     max_length=min(lowercase__		,     tokenizer.max_len      )		,     )
														all_input_ids.append(lowercase__      )

									SCREAMING_SNAKE_CASE           =     max(len(lowercase__      ) for input_ids in all_input_ids      )

									SCREAMING_SNAKE_CASE           =     []
									for ex_index, (input_ids, example) in enumerate(zip(lowercase__		,     self.examples      )      ):
														if ex_index % 10000 == 0:
																			logger.info(f'''Writing example {ex_index}/{len(self.examples      )}'''      )
														# The mask has 1 for real tokens and 0 for padding tokens. Only real
														# tokens are attended to.
														SCREAMING_SNAKE_CASE           =     [1 if mask_padding_with_zero else 0] * len(lowercase__      )

														# Zero-pad up to the sequence length.
														SCREAMING_SNAKE_CASE           =     batch_length - len(lowercase__      )
														if pad_on_left:
																			SCREAMING_SNAKE_CASE           =     ([pad_token] * padding_length) + input_ids
																			SCREAMING_SNAKE_CASE           =     ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
														else:
																			SCREAMING_SNAKE_CASE           =     input_ids + ([pad_token] * padding_length)
																			SCREAMING_SNAKE_CASE           =     attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

														if len(lowercase__      ) != batch_length:
																			raise ValueError(f'''Error with input length {len(lowercase__      )} vs {batch_length}'''      )
														if len(lowercase__      ) != batch_length:
																			raise ValueError(f'''Error with input length {len(lowercase__      )} vs {batch_length}'''      )

														if self.mode == "classification":
																			SCREAMING_SNAKE_CASE           =     label_map[example.label]
														elif self.mode == "regression":
																			SCREAMING_SNAKE_CASE           =     float(example.label      )
														else:
																			raise ValueError(self.mode      )

														if ex_index < 5 and self.verbose:
																			logger.info('*** Example ***'      )
																			logger.info(f'''guid: {example.guid}'''      )
																			logger.info(f'''input_ids: {' '.join([str(lowercase__      ) for x in input_ids]      )}'''      )
																			logger.info(f'''attention_mask: {' '.join([str(lowercase__      ) for x in attention_mask]      )}'''      )
																			logger.info(f'''label: {example.label} (id = {label})'''      )

														features.append(InputFeatures(input_ids=lowercase__		,     attention_mask=lowercase__		,     label=lowercase__      )      )

									if return_tensors is None:
														return features
									elif return_tensors == "tf":
														if not is_tf_available():
																			raise RuntimeError('return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported'      )
														import tensorflow as tf

														def gen():
																			for ex in features:
																								yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

														SCREAMING_SNAKE_CASE           =     tf.data.Dataset.from_generator(
														    lowercase__		,     ({'input_ids': tf.intaa, 'attention_mask': tf.intaa}, tf.intaa)		,     ({'input_ids': tf.TensorShape([None]      ), 'attention_mask': tf.TensorShape([None]      )}, tf.TensorShape([]      ))		,     )
														return dataset
									elif return_tensors == "pt":
														if not is_torch_available():
																			raise RuntimeError('return_tensors set to \'pt\' but PyTorch can\'t be imported'      )
														import torch
														from torch.utils.data import TensorDataset

														SCREAMING_SNAKE_CASE           =     torch.tensor([f.input_ids for f in features]		,     dtype=torch.long      )
														SCREAMING_SNAKE_CASE           =     torch.tensor([f.attention_mask for f in features]		,     dtype=torch.long      )
														if self.mode == "classification":
																			SCREAMING_SNAKE_CASE           =     torch.tensor([f.label for f in features]		,     dtype=torch.long      )
														elif self.mode == "regression":
																			SCREAMING_SNAKE_CASE           =     torch.tensor([f.label for f in features]		,     dtype=torch.float      )

														SCREAMING_SNAKE_CASE           =     TensorDataset(lowercase__		,     lowercase__		,     lowercase__      )
														return dataset
									else:
														raise ValueError('return_tensors should be one of \'tf\' or \'pt\''      )
