





import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


__A    :   str          =       get_logger(__name__)

# Datasets
__A    :   List[Any]          =       "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
__A    :   Optional[Any]          =       "https://cdn-datasets.huggingface.co/datasets/datasets"
__A    :   Union[str, Any]          =       "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"

# Metrics
__A    :   Tuple          =       "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"
__A    :   str          =       "https://cdn-datasets.huggingface.co/datasets/metric"
__A    :   Dict          =       "https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"

# Hub
__A    :   List[Any]          =       os.environ.get("HF_ENDPOINT", "https://huggingface.co")
__A    :   List[str]          =       HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
__A    :   Optional[int]          =       "main"

__A    :   Dict          =       version.parse(platform.python_version())

if PY_VERSION < version.parse("3.8"):
				import importlib_metadata
else:
				import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
__A    :   List[str]          =       {"1", "ON", "YES", "TRUE"}
__A    :   Optional[int]          =       ENV_VARS_TRUE_VALUES.union({"AUTO"})


# Imports
__A    :   Any          =       version.parse(importlib_metadata.version("dill"))
__A    :   Union[str, Any]          =       version.parse(importlib_metadata.version("pandas"))
__A    :   str          =       version.parse(importlib_metadata.version("pyarrow"))

__A    :   Optional[Any]          =       os.environ.get("USE_TF", "AUTO").upper()
__A    :   Optional[Any]          =       os.environ.get("USE_TORCH", "AUTO").upper()
__A    :   List[Any]          =       os.environ.get("USE_JAX", "AUTO").upper()

__A    :   List[Any]          =       "N/A"
__A    :   Union[str, Any]          =       False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
				__A    :   str          =       importlib.util.find_spec("torch") is not None
				if TORCH_AVAILABLE:
								try:
												__A    :   int          =       version.parse(importlib_metadata.version("torch"))
												logger.info(F'PyTorch version {TORCH_VERSION} available.')
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info("Disabling PyTorch because USE_TF is set")

__A    :   Union[str, Any]          =       "N/A"
__A    :   List[str]          =       False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
				__A    :   Optional[Any]          =       importlib.util.find_spec("tensorflow") is not None
				if TF_AVAILABLE:
								# For the metadata, we have to look for both tensorflow and tensorflow-cpu
								for package in [
								    "tensorflow",
								    "tensorflow-cpu",
								    "tensorflow-gpu",
								    "tf-nightly",
								    "tf-nightly-cpu",
								    "tf-nightly-gpu",
								    "intel-tensorflow",
								    "tensorflow-rocm",
								    "tensorflow-macos",
								]:
												try:
																__A    :   Any          =       version.parse(importlib_metadata.version(package))
												except importlib_metadata.PackageNotFoundError:
																continue
												else:
																break
								else:
												__A    :   str          =       False
				if TF_AVAILABLE:
								if TF_VERSION.major < 2:
												logger.info(F'TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.')
												__A    :   Optional[int]          =       False
								else:
												logger.info(F'TensorFlow version {TF_VERSION} available.')
else:
				logger.info("Disabling Tensorflow because USE_TORCH is set")


__A    :   Union[str, Any]          =       "N/A"
__A    :   List[str]          =       False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
				__A    :   Tuple          =       importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
				if JAX_AVAILABLE:
								try:
												__A    :   Tuple          =       version.parse(importlib_metadata.version("jax"))
												logger.info(F'JAX version {JAX_VERSION} available.')
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info("Disabling JAX because USE_JAX is set to False")


__A    :   Optional[int]          =       os.environ.get("USE_BEAM", "AUTO").upper()
__A    :   str          =       "N/A"
__A    :   int          =       False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
				try:
								__A    :   List[Any]          =       version.parse(importlib_metadata.version("apache_beam"))
								__A    :   List[str]          =       True
								logger.info(F'Apache Beam version {BEAM_VERSION} available.')
				except importlib_metadata.PackageNotFoundError:
								pass
else:
				logger.info("Disabling Apache Beam because USE_BEAM is set to False")


# Optional tools for data loading
__A    :   Dict          =       importlib.util.find_spec("sqlalchemy") is not None

# Optional tools for feature decoding
__A    :   str          =       importlib.util.find_spec("PIL") is not None
__A    :   Any          =       importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.0.31")
__A    :   Optional[Any]          =       importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.1.0")

# Optional compression tools
__A    :   Dict          =       importlib.util.find_spec("rarfile") is not None
__A    :   List[str]          =       importlib.util.find_spec("zstandard") is not None
__A    :   List[Any]          =       importlib.util.find_spec("lz4") is not None
__A    :   List[Any]          =       importlib.util.find_spec("py7zr") is not None


# Cache location
__A    :   Dict          =       "~/.cache"
__A    :   Union[str, Any]          =       os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
__A    :   str          =       os.path.join(XDG_CACHE_HOME, "huggingface")
__A    :   List[str]          =       os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))

__A    :   int          =       os.path.join(HF_CACHE_HOME, "datasets")
__A    :   Tuple          =       Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))

__A    :   Union[str, Any]          =       os.path.join(HF_CACHE_HOME, "metrics")
__A    :   Optional[Any]          =       Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE))

__A    :   int          =       os.path.join(HF_CACHE_HOME, "modules")
__A    :   int          =       Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))

__A    :   int          =       "downloads"
__A    :   Any          =       os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
__A    :   Any          =       Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))

__A    :   Union[str, Any]          =       "extracted"
__A    :   List[Any]          =       os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
__A    :   Any          =       Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
__A    :   List[Any]          =       (
    os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
__A    :   List[Any]          =       1_000

# Size of the preloaded record batch in `Dataset.__iter__`
__A    :   Dict          =       10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
__A    :   str          =       4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
__A    :   Dict          =       "500MB"

# Parquet configuration
__A    :   Optional[int]          =       100
__A    :   int          =       100
__A    :   Optional[Any]          =       100

# Offline mode
__A    :   Any          =       os.environ.get("HF_DATASETS_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES

# In-memory
__A    :   Optional[int]          =       0  # Disabled
__A    :   Dict          =       float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
__A    :   Dict          =       "dataset.arrow"
__A    :   Optional[Any]          =       "indices.arrow"
__A    :   Tuple          =       "state.json"
__A    :   Union[str, Any]          =       "dataset_info.json"
__A    :   Optional[int]          =       "dataset_infos.json"
__A    :   Optional[int]          =       "LICENSE"
__A    :   Any          =       "metric_info.json"
__A    :   Union[str, Any]          =       "dataset_dict.json"

__A    :   Optional[int]          =       "datasets_modules"

__A    :   Tuple          =       255

# Streaming
__A    :   List[str]          =       20
__A    :   str          =       5

# Datasets without script
__A    :   List[str]          =       200
__A    :   Dict          =       10
__A    :   Optional[int]          =       200

# Progress bars
__A    :   Optional[int]          =       0.05  # 20 progress updates per sec










import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    str:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file.jsonl'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    Union[str, Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_utf16_encoded.jsonl'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''			)
			with open(UpperCamelCase__      ,      '''w'''      ,      encoding='''utf-16'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    Optional[Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_with_list_of_dicts.json'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        [
            {"col_1": -1},
            {"col_1": 1, "col_2": 2},
            {"col_1": 10, "col_2": 20}
        ]
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.fixture
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__			)  ->    int:







			'''simple docstring'''

			UpperCAmelCase       		=    tmp_path / '''file_with_list_of_dicts_field.json'''
			UpperCAmelCase       		=    textwrap.dedent(
			    '''\
        {
            "field1": 1,
            "field2": "aabb",
            "field3": [
                {"col_1": -1},
                {"col_1": 1, "col_2": 2},
                {"col_1": 10, "col_2": 20}
            ]
        }
        '''			)
			with open(UpperCamelCase__      ,      '''w'''			) as f:
						f.write(UpperCamelCase__			)
			return str(UpperCamelCase__			)





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''      ,      [
        ('''jsonl_file''', {}),
        ('''jsonl_file_utf16_encoded''', {'''encoding''': '''utf-16'''}),
        ('''json_file_with_list_of_dicts''', {}),
        ('''json_file_with_list_of_dicts_field''', {'''field''': '''field3'''}),
    ]      ,      )
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__      ,      UpperCamelCase__      ,      UpperCamelCase__			)  ->    Dict:







			'''simple docstring'''

			UpperCAmelCase       		=    Json(**UpperCamelCase__			)
			UpperCAmelCase       		=    json._generate_tables([[request.getfixturevalue(UpperCamelCase__			)]]			)
			UpperCAmelCase       		=    pa.concat_tables([table for _, table in generator]			)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}






@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''      ,      [
        (
            '''jsonl_file''',
            {'''features''': Features({'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			)},
        ),
        (
            '''json_file_with_list_of_dicts''',
            {'''features''': Features({'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			)},
        ),
        (
            '''json_file_with_list_of_dicts_field''',
            {
                '''field''': '''field3''',
                '''features''': Features(
                    {'''col_1''': Value('''int64'''			), '''col_2''': Value('''int64'''			), '''missing_col''': Value('''string'''			)}			),
            },
        ),
    ]      ,      )
def 						__SCREAMING_SNAKE_CASE  (    UpperCamelCase__      ,      UpperCamelCase__      ,      UpperCamelCase__			)  ->    List[Any]:







			'''simple docstring'''

			UpperCAmelCase       		=    Json(**UpperCamelCase__			)
			UpperCAmelCase       		=    json._generate_tables([[request.getfixturevalue(UpperCamelCase__			)]]			)
			UpperCAmelCase       		=    pa.concat_tables([table for _, table in generator]			)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}



