




import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class 	_lowerCamelCase  :

      """simple docstring"""

      def __init__(       self   :	Union[str, Any]	,    snake_case   :	Optional[int]			):
          __UpperCamelCase											=  foo







      def __call__(       self   :	Optional[Any]			):
          return self.foo







class 	_lowerCamelCase  (     datasets.Dataset				):

      """simple docstring"""

      @fingerprint_transform(inplace=snake_case			)
      def  snake_case		(       self   :	Union[str, Any]	,    snake_case   :	str	,    *snake_case   :	Optional[int]	,    **snake_case   :	Dict			):
          return DatasetChild(self.data	,    fingerprint=snake_case			)







      @fingerprint_transform(inplace=snake_case			)
      def  snake_case		(       self   :	List[Any]	,    snake_case   :	str	,    *snake_case   :	int	,    **snake_case   :	Union[str, Any]			):
          return DatasetChild(self.data	,    fingerprint=snake_case			)







class 	_lowerCamelCase  :

      """simple docstring"""

      def __init__(       self   :	Any	,    snake_case   :	Optional[int]			):
          __UpperCamelCase											=  callable







      def __call__(       self   :	int	,    *snake_case   :	str	,    **snake_case   :	Union[str, Any]			):
          if self.callable is not None:
              return self.callable(*snake_case	,    **snake_case			)







      def __getstate__(       self   :	Tuple			):
          raise pickle.PicklingError()







class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

      """simple docstring"""

      @require_transformers
      @pytest.mark.integration
      def  snake_case		(       self   :	Any			):
          from transformers import AutoTokenizer

          def encode(snake_case   :	Tuple			):
              return tokenizer(snake_case			)

          # TODO: add hash consistency tests across sessions
          __UpperCamelCase											=  AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(lambda snake_case			: tokenizer(snake_case			)			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  AutoTokenizer.from_pretrained('''bert-base-cased'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(lambda snake_case			: tokenizer(snake_case			)			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(lambda snake_case			: tokenizer(snake_case			)			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      @require_transformers
      @pytest.mark.integration
      def  snake_case		(       self   :	Optional[Any]			):
          from transformers import AutoTokenizer

          __UpperCamelCase											=  AutoTokenizer.from_pretrained('''gpt2'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          tokenizer('''Hello world !'''			)  # call once to change the tokenizer's cache
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)







      @require_regex
      def  snake_case		(       self   :	Dict			):
          import regex

          __UpperCamelCase											=  regex.Regex('''foo'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  regex.Regex('''bar'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  regex.Regex('''foo'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

      """simple docstring"""

      def  snake_case		(       self   :	str			):
          def func():
              return foo

          __UpperCamelCase											=  [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  [1]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	Any			):
          def func():
              pass

          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()

          def func():
              pass

          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	str			):
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	Optional[Any]			):
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	Optional[int]			):
          __UpperCamelCase											=  (
              '''co_argcount''',
              '''co_kwonlyargcount''',
              '''co_nlocals''',
              '''co_stacksize''',
              '''co_flags''',
              '''co_code''',
              '''co_consts''',
              '''co_names''',
              '''co_varnames''',
              '''co_filename''',
              '''co_name''',
              '''co_firstlineno''',
              '''co_lnotab''',
              '''co_freevars''',
              '''co_cellvars''',
          )

          def _create_code_pyaa(*snake_case   :	Dict			):
              if hasattr(args[-3]	,    '''encode'''			):
                  __UpperCamelCase											=  list(snake_case			)
                  __UpperCamelCase											=  args[-3].encode()  # co_lnotab
                  __UpperCamelCase											=  args[-10].encode()  # co_code
              if hasattr(snake_case	,    '''co_posonlyargcount'''			):
                  if len(snake_case			) == 16:
                      return CodeType(*snake_case			)
                  elif len(snake_case			) == 15:
                      return CodeType(args[0]	,    0	,    *args[1:]			)
                  return CodeType(args[0]	,    0	,    0	,    *args[1:]			)
              elif hasattr(snake_case	,    '''co_kwonlyargcount'''			):
                  if len(snake_case			) == 16:
                      return CodeType(args[0]	,    *args[2:]			)
                  elif len(snake_case			) == 15:
                      return CodeType(*snake_case			)
                  return CodeType(args[0]	,    0	,    *args[1:]			)
              if len(snake_case			) == 16:
                  return CodeType(args[0]	,    *args[3:]			)
              elif len(snake_case			) == 15:
                  return CodeType(args[0]	,    *args[2:]			)
              return CodeType(*snake_case			)

          def create_ipython_func(snake_case   :	Optional[int]	,    snake_case   :	Optional[Any]			):
              def func():
                  return returned_obj

              __UpperCamelCase											=  func.__code__
              # Use _create_code from dill in order to make it work for different python versions
              if datasets.config.PY_VERSION < version.parse('''3.8'''			):
                  __UpperCamelCase											=  _create_code_pyaa(
                      *[getattr(snake_case	,    snake_case			) if k != '''co_filename''' else co_filename for k in code_args_pyaa]			)
              else:
                  __UpperCamelCase											=  code.replace(co_filename=snake_case			)
              return FunctionType(snake_case	,    func.__globals__	,    func.__name__	,    func.__defaults__	,    func.__closure__			)

          __UpperCamelCase						, __UpperCamelCase											=  '''<ipython-input-2-e0383a102aae>''', [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          __UpperCamelCase						, __UpperCamelCase											=  '''<ipython-input-2-e0383a102aae>''', [1]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          __UpperCamelCase						, __UpperCamelCase											=  '''<ipython-input-5-713f6613acf3>''', [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)

          __UpperCamelCase						, __UpperCamelCase											=  os.path.join(gettempdir()	,    '''ipykernel_12345'''	,    '''321456789.py'''			), [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          __UpperCamelCase						, __UpperCamelCase											=  os.path.join(gettempdir()	,    '''ipykernel_12345'''	,    '''321456789.py'''			), [1]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          __UpperCamelCase						, __UpperCamelCase											=  os.path.join(gettempdir()	,    '''ipykernel_12345'''	,    '''654123987.py'''			), [0]
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(create_ipython_func(snake_case	,    snake_case			)			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	List[Any]			):
          __UpperCamelCase						, __UpperCamelCase											=  [0], [1]

          def func():
              return foo, bar

          __UpperCamelCase											=  '''__main__'''

          def globalvars_mocka_side_effect(snake_case   :	List[str]	,    *snake_case   :	Optional[int]	,    **snake_case   :	Any			):
              return {"foo": foo, "bar": bar}

          def globalvars_mocka_side_effect(snake_case   :	Optional[Any]	,    *snake_case   :	Any	,    **snake_case   :	Any			):
              return {"bar": bar, "foo": foo}

          with patch('''dill.detect.globalvars'''	,    side_effect=snake_case			) as globalvars_mocka:
              __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
              self.assertGreater(globalvars_mocka.call_count	,    0			)
          with patch('''dill.detect.globalvars'''	,    side_effect=snake_case			) as globalvars_mocka:
              __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
              self.assertGreater(globalvars_mocka.call_count	,    0			)
          self.assertEqual(snake_case	,    snake_case			)







class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

      """simple docstring"""

      def  snake_case		(       self   :	Optional[int]			):
          __UpperCamelCase											=  Hasher.hash('''hello'''			)
          __UpperCamelCase											=  Hasher.hash('''hello'''			)
          __UpperCamelCase											=  Hasher.hash('''there'''			)
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	List[str]			):
          __UpperCamelCase											=  Hasher.hash(Foo('''hello'''			)			)
          __UpperCamelCase											=  Hasher.hash(Foo('''hello'''			)			)
          __UpperCamelCase											=  Hasher.hash(Foo('''there'''			)			)
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	List[Any]			):
          __UpperCamelCase											=  Hasher()
          for x in ["hello", Foo('''hello'''			)]:
              hasher.update(snake_case			)
          __UpperCamelCase											=  hasher.hexdigest()
          __UpperCamelCase											=  Hasher()
          for x in ["hello", Foo('''hello'''			)]:
              hasher.update(snake_case			)
          __UpperCamelCase											=  hasher.hexdigest()
          __UpperCamelCase											=  Hasher()
          for x in ["there", Foo('''there'''			)]:
              hasher.update(snake_case			)
          __UpperCamelCase											=  hasher.hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      def  snake_case		(       self   :	List[str]			):
          with self.assertRaises(pickle.PicklingError			):
              Hasher.hash(UnpicklableCallable(Foo('''hello'''			)			)			)







      def  snake_case		(       self   :	List[str]			):
          __UpperCamelCase											=  '''abc'''
          __UpperCamelCase											=  [string, string]  # two strings have the same ids
          __UpperCamelCase											=  [string, string]
          __UpperCamelCase											=  json.loads(F"[\"{string}\", \"{string}\"]"			)  # two strings have different ids
          self.assertIs(obja[0]	,    snake_case			)
          self.assertIs(obja[0]	,    obja[1]			)
          self.assertIs(obja[0]	,    snake_case			)
          self.assertIs(obja[0]	,    obja[1]			)
          self.assertIsNot(obja[0]	,    snake_case			)
          self.assertIsNot(obja[0]	,    obja[1]			)
          __UpperCamelCase											=  Hasher.hash(snake_case			)
          __UpperCamelCase											=  Hasher.hash(snake_case			)
          __UpperCamelCase											=  Hasher.hash(snake_case			)
          self.assertEqual(snake_case	,    snake_case			)
          self.assertEqual(snake_case	,    snake_case			)







      @require_tiktoken
      def  snake_case		(       self   :	int			):
          import tiktoken

          __UpperCamelCase											=  tiktoken.get_encoding('''gpt2'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  tiktoken.get_encoding('''r50k_base'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  tiktoken.get_encoding('''gpt2'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      @require_torch
      def  snake_case		(       self   :	Optional[Any]			):
          import torch

          __UpperCamelCase											=  torch.tensor([1.0]			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  torch.tensor([2.0]			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  torch.tensor([1.0]			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







      @require_spacy
      @require_spacy_model('''en_core_web_sm'''			)
      @require_spacy_model('''fr_core_news_sm'''			)
      @pytest.mark.integration
      def  snake_case		(       self   :	List[str]			):
          import spacy

          __UpperCamelCase											=  spacy.load('''en_core_web_sm'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  spacy.load('''fr_core_news_sm'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          __UpperCamelCase											=  spacy.load('''en_core_web_sm'''			)
          __UpperCamelCase											=  mda(datasets.utils.py_utils.dumps(snake_case			)			).hexdigest()
          self.assertEqual(snake_case	,    snake_case			)
          self.assertNotEqual(snake_case	,    snake_case			)







@pytest.mark.integration
def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		Any:

    """simple docstring"""


    __UpperCamelCase											=  tmp_path / '''dir1'''
    __UpperCamelCase											=  tmp_path / '''dir2'''
    dira.mkdir()
    dira.mkdir()
    __UpperCamelCase											=  '''script.py'''
    __UpperCamelCase											=  dedent(
        '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    ''' )
    __UpperCamelCase											=  dira / script_filename
    __UpperCamelCase											=  dira / script_filename
    with script_patha.open('''w''' ) as f:
        f.write(lowercase_ )
    with script_patha.open('''w''' ) as f:
        f.write(lowercase_ )
    __UpperCamelCase											=  subprocess.check_output(['''python''', str(lowercase_ )] )
    __UpperCamelCase											=  subprocess.check_output(['''python''', str(lowercase_ )] )
    assert fingerprinta == fingerprinta




def    __SCREAMING_SNAKE_CASE				(					)    ->		Optional[Any]:

    """simple docstring"""


    __UpperCamelCase											=  {'''a''': [0, 1, 2]}
    __UpperCamelCase											=  DatasetChild(InMemoryTable.from_pydict(lowercase_ ) )
    __UpperCamelCase											=  dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2 ) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca ).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca ).get()._fingerprint




def    __SCREAMING_SNAKE_CASE				(					)    ->		Tuple:

    """simple docstring"""


    __UpperCamelCase											=  {'''a''': [0, 1, 2]}

    class 	_lowerCamelCase  (     datasets.Dataset				):

          """simple docstring"""

          @fingerprint_transform(inplace=snake_case			)
          def  snake_case		(       self   :	List[str]	,    snake_case   :	Any			):
              return DummyDatasetChild(self.data	,    fingerprint=snake_case			)







    __UpperCamelCase											=  DummyDatasetChild(InMemoryTable.from_pydict(lowercase_ ) ).func()

    class 	_lowerCamelCase  (     datasets.Dataset				):

          """simple docstring"""

          @fingerprint_transform(inplace=snake_case	,    version='''1.0.0'''			)
          def  snake_case		(       self   :	Union[str, Any]	,    snake_case   :	str			):
              return DummyDatasetChild(self.data	,    fingerprint=snake_case			)







    __UpperCamelCase											=  DummyDatasetChild(InMemoryTable.from_pydict(lowercase_ ) ).func()

    class 	_lowerCamelCase  (     datasets.Dataset				):

          """simple docstring"""

          @fingerprint_transform(inplace=snake_case	,    version='''2.0.0'''			)
          def  snake_case		(       self   :	str	,    snake_case   :	str			):
              return DummyDatasetChild(self.data	,    fingerprint=snake_case			)







    __UpperCamelCase											=  DummyDatasetChild(InMemoryTable.from_pydict(lowercase_ ) ).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a} ) == 3




def    __SCREAMING_SNAKE_CASE				(					)    ->		Tuple:

    """simple docstring"""


    __UpperCamelCase											=  Hasher()
    hasher.update(lambda lowercase_ : x )












# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


a_										=       "Here is a text containing a lot of information: '''{text}'''.\n\nCan you answer this question about the text: '{question}'"

class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

      """simple docstring"""

      lowerCAmelCase__     :	Union[str, Any]      =   "google/flan-t5-base"
      lowerCAmelCase__     :	Union[str, Any]      =   (
          "This is a tool that answers questions related to a text. It takes two arguments named `text`, which is the "
          "text where to find the answer, and `question`, which is the question, and returns the answer to the question."
      )
      lowerCAmelCase__     :	List[Any]      =   "text_qa"
      lowerCAmelCase__     :	int      =   AutoTokenizer
      lowerCAmelCase__     :	Optional[Any]      =   AutoModelForSeqaSeqLM

      lowerCAmelCase__     :	Tuple      =   ["text", "text"]
      lowerCAmelCase__     :	Any      =   ["text"]







      def  snake_case		(       self   :	Union[str, Any]	,    snake_case   :	str	,    snake_case   :	str			):
          __UpperCamelCase											=  QA_PROMPT.format(text=snake_case	,    question=snake_case			)
          return self.pre_processor(snake_case	,    return_tensors='''pt'''			)







      def  snake_case		(       self   :	Any	,    snake_case   :	Optional[int]			):
          __UpperCamelCase											=  self.model.generate(**snake_case			)

          __UpperCamelCase						, __UpperCamelCase											=  inputs['''input_ids'''].shape
          __UpperCamelCase											=  output_ids.shape[0]

          return output_ids.reshape(snake_case	,    out_b // in_b	,    *output_ids.shape[1:]			)[0][0]







      def  snake_case		(       self   :	Tuple	,    snake_case   :	Optional[Any]			):
          return self.pre_processor.decode(snake_case	,    skip_special_tokens=snake_case	,    clean_up_tokenization_spaces=snake_case			)






