




import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


__SCREAMING_SNAKE_CASE      :				str				      =get_logger(__name__)

# Datasets
__SCREAMING_SNAKE_CASE      :				Tuple				      ="""https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"""
__SCREAMING_SNAKE_CASE      :				str				      ="""https://cdn-datasets.huggingface.co/datasets/datasets"""
__SCREAMING_SNAKE_CASE      :				List[Any]				      ="""https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"""

# Metrics
__SCREAMING_SNAKE_CASE      :				int				      ="""https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"""
__SCREAMING_SNAKE_CASE      :				str				      ="""https://cdn-datasets.huggingface.co/datasets/metric"""
__SCREAMING_SNAKE_CASE      :				Tuple				      ="""https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"""

# Hub
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =os.environ.get("""HF_ENDPOINT""", """https://huggingface.co""")
__SCREAMING_SNAKE_CASE      :				List[Any]				      =HF_ENDPOINT + """/datasets/{repo_id}/resolve/{revision}/{path}"""
__SCREAMING_SNAKE_CASE      :				str				      ="""main"""

__SCREAMING_SNAKE_CASE      :				Optional[int]				      =version.parse(platform.python_version())

if PY_VERSION < version.parse("""3.8"""):
      import importlib_metadata
else:
      import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
__SCREAMING_SNAKE_CASE      :				List[Any]				      ={"""1""", """ON""", """YES""", """TRUE"""}
__SCREAMING_SNAKE_CASE      :				Optional[int]				      =ENV_VARS_TRUE_VALUES.union({"""AUTO"""})


# Imports
__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      =version.parse(importlib_metadata.version("""dill"""))
__SCREAMING_SNAKE_CASE      :				str				      =version.parse(importlib_metadata.version("""pandas"""))
__SCREAMING_SNAKE_CASE      :				int				      =version.parse(importlib_metadata.version("""pyarrow"""))

__SCREAMING_SNAKE_CASE      :				Tuple				      =os.environ.get("""USE_TF""", """AUTO""").upper()
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =os.environ.get("""USE_TORCH""", """AUTO""").upper()
__SCREAMING_SNAKE_CASE      :				Tuple				      =os.environ.get("""USE_JAX""", """AUTO""").upper()

__SCREAMING_SNAKE_CASE      :				List[str]				      ="""N/A"""
__SCREAMING_SNAKE_CASE      :				Optional[int]				      =False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
      __SCREAMING_SNAKE_CASE      :				List[Any]				      =importlib.util.find_spec("""torch""") is not None
      if TORCH_AVAILABLE:
            try:
                  __SCREAMING_SNAKE_CASE      :				Optional[int]				      =version.parse(importlib_metadata.version("""torch"""))
                  logger.info(F"""PyTorch version {TORCH_VERSION} available.""")
            except importlib_metadata.PackageNotFoundError:
                  pass
else:
      logger.info("""Disabling PyTorch because USE_TF is set""")

__SCREAMING_SNAKE_CASE      :				List[Any]				      ="""N/A"""
__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      =False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
      __SCREAMING_SNAKE_CASE      :				List[Any]				      =importlib.util.find_spec("""tensorflow""") is not None
      if TF_AVAILABLE:
            # For the metadata, we have to look for both tensorflow and tensorflow-cpu
            for package in [
                "tensorflow",
                "tensorflow-cpu",
                "tensorflow-gpu",
                "tf-nightly",
                "tf-nightly-cpu",
                "tf-nightly-gpu",
                "intel-tensorflow",
                "tensorflow-rocm",
                "tensorflow-macos",
            ]:
                  try:
                        __SCREAMING_SNAKE_CASE      :				Optional[int]				      =version.parse(importlib_metadata.version(package))
                  except importlib_metadata.PackageNotFoundError:
                        continue
                  else:
                        break
            else:
                  __SCREAMING_SNAKE_CASE      :				str				      =False
      if TF_AVAILABLE:
            if TF_VERSION.major < 2:
                  logger.info(F"""TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.""")
                  __SCREAMING_SNAKE_CASE      :				Dict				      =False
            else:
                  logger.info(F"""TensorFlow version {TF_VERSION} available.""")
else:
      logger.info("""Disabling Tensorflow because USE_TORCH is set""")


__SCREAMING_SNAKE_CASE      :				List[Any]				      ="""N/A"""
__SCREAMING_SNAKE_CASE      :				Any				      =False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
      __SCREAMING_SNAKE_CASE      :				List[str]				      =importlib.util.find_spec("""jax""") is not None and importlib.util.find_spec("""jaxlib""") is not None
      if JAX_AVAILABLE:
            try:
                  __SCREAMING_SNAKE_CASE      :				int				      =version.parse(importlib_metadata.version("""jax"""))
                  logger.info(F"""JAX version {JAX_VERSION} available.""")
            except importlib_metadata.PackageNotFoundError:
                  pass
else:
      logger.info("""Disabling JAX because USE_JAX is set to False""")


__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =os.environ.get("""USE_BEAM""", """AUTO""").upper()
__SCREAMING_SNAKE_CASE      :				List[str]				      ="""N/A"""
__SCREAMING_SNAKE_CASE      :				Dict				      =False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
      try:
            __SCREAMING_SNAKE_CASE      :				List[Any]				      =version.parse(importlib_metadata.version("""apache_beam"""))
            __SCREAMING_SNAKE_CASE      :				str				      =True
            logger.info(F"""Apache Beam version {BEAM_VERSION} available.""")
      except importlib_metadata.PackageNotFoundError:
            pass
else:
      logger.info("""Disabling Apache Beam because USE_BEAM is set to False""")


# Optional tools for data loading
__SCREAMING_SNAKE_CASE      :				int				      =importlib.util.find_spec("""sqlalchemy""") is not None

# Optional tools for feature decoding
__SCREAMING_SNAKE_CASE      :				List[str]				      =importlib.util.find_spec("""PIL""") is not None
__SCREAMING_SNAKE_CASE      :				Tuple				      =importlib.util.find_spec("""soundfile""") is not None and version.parse(
    importlib.import_module("""soundfile""").__libsndfile_version__
) >= version.parse("""1.0.31""")
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =importlib.util.find_spec("""soundfile""") is not None and version.parse(
    importlib.import_module("""soundfile""").__libsndfile_version__
) >= version.parse("""1.1.0""")

# Optional compression tools
__SCREAMING_SNAKE_CASE      :				List[Any]				      =importlib.util.find_spec("""rarfile""") is not None
__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      =importlib.util.find_spec("""zstandard""") is not None
__SCREAMING_SNAKE_CASE      :				str				      =importlib.util.find_spec("""lz4""") is not None
__SCREAMING_SNAKE_CASE      :				str				      =importlib.util.find_spec("""py7zr""") is not None


# Cache location
__SCREAMING_SNAKE_CASE      :				Dict				      ="""~/.cache"""
__SCREAMING_SNAKE_CASE      :				Dict				      =os.getenv("""XDG_CACHE_HOME""", DEFAULT_XDG_CACHE_HOME)
__SCREAMING_SNAKE_CASE      :				str				      =os.path.join(XDG_CACHE_HOME, """huggingface""")
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =os.path.expanduser(os.getenv("""HF_HOME""", DEFAULT_HF_CACHE_HOME))

__SCREAMING_SNAKE_CASE      :				List[str]				      =os.path.join(HF_CACHE_HOME, """datasets""")
__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      =Path(os.getenv("""HF_DATASETS_CACHE""", DEFAULT_HF_DATASETS_CACHE))

__SCREAMING_SNAKE_CASE      :				str				      =os.path.join(HF_CACHE_HOME, """metrics""")
__SCREAMING_SNAKE_CASE      :				Tuple				      =Path(os.getenv("""HF_METRICS_CACHE""", DEFAULT_HF_METRICS_CACHE))

__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =os.path.join(HF_CACHE_HOME, """modules""")
__SCREAMING_SNAKE_CASE      :				Tuple				      =Path(os.getenv("""HF_MODULES_CACHE""", DEFAULT_HF_MODULES_CACHE))

__SCREAMING_SNAKE_CASE      :				Dict				      ="""downloads"""
__SCREAMING_SNAKE_CASE      :				List[str]				      =os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
__SCREAMING_SNAKE_CASE      :				int				      =Path(os.getenv("""HF_DATASETS_DOWNLOADED_DATASETS_PATH""", DEFAULT_DOWNLOADED_DATASETS_PATH))

__SCREAMING_SNAKE_CASE      :				int				      ="""extracted"""
__SCREAMING_SNAKE_CASE      :				List[Any]				      =os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
__SCREAMING_SNAKE_CASE      :				Any				      =Path(os.getenv("""HF_DATASETS_EXTRACTED_DATASETS_PATH""", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =(
    os.environ.get("""HF_UPDATE_DOWNLOAD_COUNTS""", """AUTO""").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
__SCREAMING_SNAKE_CASE      :				int				      =1_000

# Size of the preloaded record batch in `Dataset.__iter__`
__SCREAMING_SNAKE_CASE      :				Dict				      =10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
__SCREAMING_SNAKE_CASE      :				List[Any]				      =4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
__SCREAMING_SNAKE_CASE      :				List[Any]				      ="""500MB"""

# Parquet configuration
__SCREAMING_SNAKE_CASE      :				List[Any]				      =100
__SCREAMING_SNAKE_CASE      :				str				      =100
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =100

# Offline mode
__SCREAMING_SNAKE_CASE      :				str				      =os.environ.get("""HF_DATASETS_OFFLINE""", """AUTO""").upper() in ENV_VARS_TRUE_VALUES

# In-memory
__SCREAMING_SNAKE_CASE      :				Dict				      =0  # Disabled
__SCREAMING_SNAKE_CASE      :				List[str]				      =float(os.environ.get("""HF_DATASETS_IN_MEMORY_MAX_SIZE""", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
__SCREAMING_SNAKE_CASE      :				Optional[int]				      ="""dataset.arrow"""
__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      ="""indices.arrow"""
__SCREAMING_SNAKE_CASE      :				List[str]				      ="""state.json"""
__SCREAMING_SNAKE_CASE      :				Any				      ="""dataset_info.json"""
__SCREAMING_SNAKE_CASE      :				int				      ="""dataset_infos.json"""
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      ="""LICENSE"""
__SCREAMING_SNAKE_CASE      :				int				      ="""metric_info.json"""
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      ="""dataset_dict.json"""

__SCREAMING_SNAKE_CASE      :				Any				      ="""datasets_modules"""

__SCREAMING_SNAKE_CASE      :				Tuple				      =255

# Streaming
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =20
__SCREAMING_SNAKE_CASE      :				List[str]				      =5

# Datasets without script
__SCREAMING_SNAKE_CASE      :				List[str]				      =200
__SCREAMING_SNAKE_CASE      :				Optional[Any]				      =10
__SCREAMING_SNAKE_CASE      :				Optional[int]				      =200

# Progress bars
__SCREAMING_SNAKE_CASE      :				List[str]				      =0.05  # 20 progress updates per sec





import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    SummarizationPipeline,
    TFPreTrainedModel,
    pipeline,
)
from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
from transformers.tokenization_utils import TruncationStrategy

from .test_pipelines_common import ANY


__SCREAMING_SNAKE_CASE      :				Union[str, Any]				      =-1 if torch_device == """cpu""" else 0






@is_pipeline_test
class      A     (						unittest.TestCase						):
     """simple docstring"""

     _snake_case:		Tuple       =MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
     _snake_case:		int       =TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING




     def 	__snake_case				(      self					:				Any		,					A__					:				str		,					A__					:				int		,					A__					:				Dict     )			->    Any:
           lowerCamelCase__ 					=			SummarizationPipeline(model=A__		,					tokenizer=A__     )
           return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]




     def 	__snake_case				(      self					:				int		,					A__					:				int		,					A__					:				Tuple     )			->    Tuple:
           lowerCamelCase__ 					=			summarizer.model

           lowerCamelCase__ 					=			summarizer('(CNN)The Palestinian Authority officially became'     )
           self.assertEqual(A__		,					[{'summary_text': ANY(A__     )}]     )

           lowerCamelCase__ 					=			summarizer(
               '(CNN)The Palestinian Authority officially became '		,					num_beams=2		,					min_length=2		,					max_length=5		,					)
           self.assertEqual(A__		,					[{'summary_text': ANY(A__     )}]     )

           # Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
           lowerCamelCase__ 					=			[
               'SwitchTransformersConfig',
               'T5Config',
               'LongT5Config',
               'LEDConfig',
               'PegasusXConfig',
               'FSMTConfig',
               'M2M100Config',
               'ProphetNetConfig',  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
           ]
           if model.config.__class__.__name__ not in model_can_handle_longer_seq:
                 # Too long and exception is expected.
                 # For TF models, if the weights are initialized in GPU context, we won't get expected index error from
                 # the embedding layer.
                 if not (
                     isinstance(A__		,					A__     )
                     and get_gpu_count() > 0
                     and len(summarizer.model.trainable_weights     ) > 0
                 ):
                       with self.assertRaises(A__     ):
                             lowerCamelCase__ 					=			summarizer('This ' * 1_0_0_0     )
           lowerCamelCase__ 					=			summarizer('This ' * 1_0_0_0		,					truncation=TruncationStrategy.ONLY_FIRST     )




     @require_torch
     def 	__snake_case				(      self					:				str     )			->    Union[str, Any]:
           lowerCamelCase__ 					=			pipeline(task='summarization'		,					model='sshleifer/tiny-mbart'		,					framework='pt'     )
           lowerCamelCase__ 					=			summarizer('This is a small test'     )
           self.assertEqual(
               A__		,					[
                   {
                       'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
                   }
               ]		,					)




     @require_tf
     def 	__snake_case				(      self					:				List[str]     )			->    List[str]:
           lowerCamelCase__ 					=			pipeline(task='summarization'		,					model='sshleifer/tiny-mbart'		,					framework='tf'     )
           lowerCamelCase__ 					=			summarizer('This is a small test'     )
           self.assertEqual(
               A__		,					[
                   {
                       'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
                   }
               ]		,					)




     @require_torch
     @slow
     def 	__snake_case				(      self					:				List[str]     )			->    Tuple:
           lowerCamelCase__ 					=			pipeline(task='summarization'		,					device=A__     )
           lowerCamelCase__ 					=			(
               ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on'
               ' Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The'
               ' formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based.'
               ' The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its'
               ' jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East'
               ' Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the'
               ' situation in Palestinian territories, paving the way for possible war crimes investigations against'
               ' Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and'
               ' the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the'
               ' body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a'
               ' move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the'
               ' world is also a step closer to ending a long era of impunity and injustice," he said, according to an'
               ' ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge'
               ' Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the'
               ' Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine'
               ' acquires all the rights as well as responsibilities that come with being a State Party to the Statute.'
               ' These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights'
               ' Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should'
               ' immediately end their pressure, and countries that support universal acceptance of the court\'s treaty'
               ' should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the'
               ' group. \"What\'s objectionable is the attempts to undermine international justice, not Palestine\'s'
               ' decision to join a treaty to which over 100 countries around the world are members." In January, when'
               ' the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an'
               ' outrage, saying the court was overstepping its boundaries. The United States also said it "strongly"'
               ' disagreed with the court\'s decision. \"As we have said repeatedly, we do not believe that Palestine is a'
               ' state and therefore we do not believe that it is eligible to join the ICC," the State Department said in'
               ' a statement. It urged the warring sides to resolve their differences through direct negotiations. "We'
               ' will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,"'
               ' it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the'
               ' territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the'
               ' court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou'
               ' Bensouda said her office would "conduct its analysis in full independence and impartiality." The war'
               ' between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry'
               ' will include alleged war crimes committed since June. The International Criminal Court was set up in'
               ' 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder'
               ' and Faith Karimi contributed to this report.'
           )
           lowerCamelCase__ 					=			(
               ' The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives'
               ' the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States'
               ' opposed the Palestinians\' efforts to join the court . Rights group Human Rights Watch welcomes the move,'
               ' says governments seeking to penalize Palestine should end pressure .'
           )
           lowerCamelCase__ 					=			summarizer(A__     )
           self.assertEqual(result[0]['summary_text']		,					A__     )