






import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


A__     :      List[Any]             =			get_logger(__name__)

# Datasets
A__     :      Union[str, Any]             =			'https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'
A__     :      Dict             =			'https://cdn-datasets.huggingface.co/datasets/datasets'
A__     :      str             =			'https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'

# Metrics
A__     :      Optional[int]             =			'https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'
A__     :      Optional[Any]             =			'https://cdn-datasets.huggingface.co/datasets/metric'
A__     :      Tuple             =			'https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'

# Hub
A__     :      int             =			os.environ.get('HF_ENDPOINT', 'https://huggingface.co')
A__     :      str             =			HF_ENDPOINT + '/datasets/{repo_id}/resolve/{revision}/{path}'
A__     :      Any             =			'main'

A__     :      Tuple             =			version.parse(platform.python_version())

if PY_VERSION < version.parse('3.8'):
							import importlib_metadata
else:
							import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
A__     :      List[str]             =			{'1', 'ON', 'YES', 'TRUE'}
A__     :      List[Any]             =			ENV_VARS_TRUE_VALUES.union({'AUTO'})


# Imports
A__     :      List[str]             =			version.parse(importlib_metadata.version('dill'))
A__     :      int             =			version.parse(importlib_metadata.version('pandas'))
A__     :      Optional[Any]             =			version.parse(importlib_metadata.version('pyarrow'))

A__     :      Union[str, Any]             =			os.environ.get('USE_TF', 'AUTO').upper()
A__     :      List[Any]             =			os.environ.get('USE_TORCH', 'AUTO').upper()
A__     :      Optional[Any]             =			os.environ.get('USE_JAX', 'AUTO').upper()

A__     :      List[Any]             =			'N/A'
A__     :      int             =			False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
							A__     :      List[str]             =			importlib.util.find_spec('torch') is not None
							if TORCH_AVAILABLE:
														try:
																					A__     :      Union[str, Any]             =			version.parse(importlib_metadata.version('torch'))
																					logger.info(F"PyTorch version {TORCH_VERSION} available.")
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info('Disabling PyTorch because USE_TF is set')

A__     :      int             =			'N/A'
A__     :      Optional[int]             =			False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
							A__     :      Optional[int]             =			importlib.util.find_spec('tensorflow') is not None
							if TF_AVAILABLE:
														# For the metadata, we have to look for both tensorflow and tensorflow-cpu
														for package in [
														    "tensorflow",
														    "tensorflow-cpu",
														    "tensorflow-gpu",
														    "tf-nightly",
														    "tf-nightly-cpu",
														    "tf-nightly-gpu",
														    "intel-tensorflow",
														    "tensorflow-rocm",
														    "tensorflow-macos",
														]:
																					try:
																												A__     :      Any             =			version.parse(importlib_metadata.version(package))
																					except importlib_metadata.PackageNotFoundError:
																												continue
																					else:
																												break
														else:
																					A__     :      Tuple             =			False
							if TF_AVAILABLE:
														if TF_VERSION.major < 2:
																					logger.info(F"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
																					A__     :      Any             =			False
														else:
																					logger.info(F"TensorFlow version {TF_VERSION} available.")
else:
							logger.info('Disabling Tensorflow because USE_TORCH is set')


A__     :      Union[str, Any]             =			'N/A'
A__     :      str             =			False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
							A__     :      Union[str, Any]             =			importlib.util.find_spec('jax') is not None and importlib.util.find_spec('jaxlib') is not None
							if JAX_AVAILABLE:
														try:
																					A__     :      str             =			version.parse(importlib_metadata.version('jax'))
																					logger.info(F"JAX version {JAX_VERSION} available.")
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info('Disabling JAX because USE_JAX is set to False')


A__     :      str             =			os.environ.get('USE_BEAM', 'AUTO').upper()
A__     :      List[Any]             =			'N/A'
A__     :      Tuple             =			False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
							try:
														A__     :      str             =			version.parse(importlib_metadata.version('apache_beam'))
														A__     :      Any             =			True
														logger.info(F"Apache Beam version {BEAM_VERSION} available.")
							except importlib_metadata.PackageNotFoundError:
														pass
else:
							logger.info('Disabling Apache Beam because USE_BEAM is set to False')


# Optional tools for data loading
A__     :      Optional[int]             =			importlib.util.find_spec('sqlalchemy') is not None

# Optional tools for feature decoding
A__     :      List[str]             =			importlib.util.find_spec('PIL') is not None
A__     :      Optional[Any]             =			importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.0.31')
A__     :      Tuple             =			importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.1.0')

# Optional compression tools
A__     :      List[Any]             =			importlib.util.find_spec('rarfile') is not None
A__     :      Union[str, Any]             =			importlib.util.find_spec('zstandard') is not None
A__     :      Any             =			importlib.util.find_spec('lz4') is not None
A__     :      int             =			importlib.util.find_spec('py7zr') is not None


# Cache location
A__     :      Tuple             =			'~/.cache'
A__     :      Dict             =			os.getenv('XDG_CACHE_HOME', DEFAULT_XDG_CACHE_HOME)
A__     :      Dict             =			os.path.join(XDG_CACHE_HOME, 'huggingface')
A__     :      Tuple             =			os.path.expanduser(os.getenv('HF_HOME', DEFAULT_HF_CACHE_HOME))

A__     :      str             =			os.path.join(HF_CACHE_HOME, 'datasets')
A__     :      List[Any]             =			Path(os.getenv('HF_DATASETS_CACHE', DEFAULT_HF_DATASETS_CACHE))

A__     :      Optional[int]             =			os.path.join(HF_CACHE_HOME, 'metrics')
A__     :      Optional[int]             =			Path(os.getenv('HF_METRICS_CACHE', DEFAULT_HF_METRICS_CACHE))

A__     :      List[str]             =			os.path.join(HF_CACHE_HOME, 'modules')
A__     :      List[Any]             =			Path(os.getenv('HF_MODULES_CACHE', DEFAULT_HF_MODULES_CACHE))

A__     :      Optional[int]             =			'downloads'
A__     :      Optional[int]             =			os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
A__     :      int             =			Path(os.getenv('HF_DATASETS_DOWNLOADED_DATASETS_PATH', DEFAULT_DOWNLOADED_DATASETS_PATH))

A__     :      Optional[int]             =			'extracted'
A__     :      Union[str, Any]             =			os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
A__     :      Tuple             =			Path(os.getenv('HF_DATASETS_EXTRACTED_DATASETS_PATH', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
A__     :      List[str]             =			(
    os.environ.get('HF_UPDATE_DOWNLOAD_COUNTS', 'AUTO').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
A__     :      List[Any]             =			10_00

# Size of the preloaded record batch in `Dataset.__iter__`
A__     :      int             =			10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
A__     :      str             =			4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
A__     :      Dict             =			'500MB'

# Parquet configuration
A__     :      Optional[Any]             =			1_00
A__     :      List[Any]             =			1_00
A__     :      Tuple             =			1_00

# Offline mode
A__     :      Optional[Any]             =			os.environ.get('HF_DATASETS_OFFLINE', 'AUTO').upper() in ENV_VARS_TRUE_VALUES

# In-memory
A__     :      Union[str, Any]             =			0  # Disabled
A__     :      Tuple             =			float(os.environ.get('HF_DATASETS_IN_MEMORY_MAX_SIZE', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
A__     :      str             =			'dataset.arrow'
A__     :      List[str]             =			'indices.arrow'
A__     :      List[Any]             =			'state.json'
A__     :      int             =			'dataset_info.json'
A__     :      Tuple             =			'dataset_infos.json'
A__     :      int             =			'LICENSE'
A__     :      Union[str, Any]             =			'metric_info.json'
A__     :      List[Any]             =			'dataset_dict.json'

A__     :      Optional[Any]             =			'datasets_modules'

A__     :      Dict             =			2_55

# Streaming
A__     :      Optional[Any]             =			20
A__     :      str             =			5

# Datasets without script
A__     :      Optional[int]             =			2_00
A__     :      Optional[Any]             =			10
A__     :      Any             =			2_00

# Progress bars
A__     :      Any             =			0.05  # 20 progress updates per sec












import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil






@pytest.fixture
def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      tmp_path / '''text.txt'''
				lowercase__       			=      textwrap.dedent(
				    '''\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        '''    )
				with open(lowerCamelCase_		,       '''w'''		,       encoding='''utf-8'''    ) as f:
								f.write(lowerCamelCase_    )
				return str(lowerCamelCase_    )






@pytest.fixture
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      tmp_path / '''text_with_image.txt'''
				with open(lowerCamelCase_		,       '''w'''		,       encoding='''utf-8'''    ) as f:
								f.write(lowerCamelCase_    )
				return str(lowerCamelCase_    )






@pytest.mark.parametrize('''keep_linebreaks'''		,       [True, False]    )
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read().splitlines(keepends=lowerCamelCase_    )
				lowercase__       			=      Text(keep_linebreaks=lowerCamelCase_		,       encoding='''utf-8'''    )
				lowercase__       			=      text._generate_tables([[text_file]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    ).to_pydict()['''text''']
				assert generated_content == expected_content






@require_pil
def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read().splitlines()[0]
				lowercase__       			=      Text(encoding='''utf-8'''		,       features=Features({'''image''': Image()}    )    )
				lowercase__       			=      text._generate_tables([[text_file_with_image]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    )
				assert pa_table.schema.field('''image'''    ).type == Image()()
				lowercase__       			=      pa_table.to_pydict()['''image''']
				assert generated_content == [{"path": image_file, "bytes": None}]






@pytest.mark.parametrize('''sample_by'''		,       ['''line''', '''paragraph''', '''document''']    )
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read()
				if sample_by == "line":
								lowercase__       			=      expected_content.splitlines()
				elif sample_by == "paragraph":
								lowercase__       			=      expected_content.split('''\n\n'''    )
				elif sample_by == "document":
								lowercase__       			=      [expected_content]
				lowercase__       			=      Text(sample_by=lowerCamelCase_		,       encoding='''utf-8'''		,       chunksize=100    )
				lowercase__       			=      text._generate_tables([[text_file]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    ).to_pydict()['''text''']
				assert generated_content == expected_content




