




import numpy as np







def 			lowerCAmelCase  (     UpperCAmelCase,     UpperCAmelCase="utf-8",     UpperCAmelCase="surrogatepass"	)  ->Tuple:






					"""simple docstring"""






					__magic_name__ :							Union[str, Any]		       =							bin(int.from_bytes(text.encode(UpperCAmelCase,     UpperCAmelCase	),     '''big'''	)	)[2:]
					return bits.zfill(8 * ((len(UpperCAmelCase	) + 7) // 8)	)







def 			lowerCAmelCase  (     UpperCAmelCase,     UpperCAmelCase="utf-8",     UpperCAmelCase="surrogatepass"	)  ->Optional[Any]:






					"""simple docstring"""






					__magic_name__ :							Dict		       =							int(UpperCAmelCase,     2	)
					return n.to_bytes((n.bit_length() + 7) // 8,     '''big'''	).decode(UpperCAmelCase,     UpperCAmelCase	) or "\0"







def 			lowerCAmelCase  (     UpperCAmelCase,     UpperCAmelCase	)  ->Dict:






					"""simple docstring"""






					if size_par + len(UpperCAmelCase	) <= 2**size_par - (len(UpperCAmelCase	) - 1):
										raise ValueError('''size of parity don\'t match with size of data'''	)

					__magic_name__ :							Any		       =							[]
					__magic_name__ :							Optional[int]		       =							[]
					__magic_name__ :							List[str]		       =							[bin(UpperCAmelCase	)[2:] for x in range(1,     size_par + len(UpperCAmelCase	) + 1	)]

					# sorted information data for the size of the output data
					__magic_name__ :							Optional[Any]		       =							[]
					# data position template + parity
					__magic_name__ :							Union[str, Any]		       =							[]
					# parity bit counter
					__magic_name__ :							int		       =							0
					# counter position of data bits
					__magic_name__ :							Dict		       =							0

					for x in range(1,     size_par + len(UpperCAmelCase	) + 1	):
										# Performs a template of bit positions - who should be given,
										# and who should be parity
										if qtd_bp < size_par:
															if (np.log(UpperCAmelCase	) / np.log(2	)).is_integer():
																				data_out_gab.append('''P'''	)
																				__magic_name__ :							Optional[Any]		       =							qtd_bp + 1
															else:
																				data_out_gab.append('''D'''	)
										else:
															data_out_gab.append('''D'''	)

										# Sorts the data to the new output size
										if data_out_gab[-1] == "D":
															data_ord.append(data[cont_data]	)
															cont_data += 1
										else:
															data_ord.append(UpperCAmelCase	)

    # Calculates parity
					__magic_name__ :							List[Any]		       =							0  # parity bit counter
					for bp in range(1,     size_par + 1	):
										# Bit counter one for a given parity
										__magic_name__ :							List[str]		       =							0
										# counter to control the loop reading
										__magic_name__ :							Any		       =							0
										for x in data_ord:
															if x is not None:
																				try:
																									__magic_name__ :							Any		       =							(bin_pos[cont_loop])[-1 * (bp)]
																				except IndexError:
																									__magic_name__ :							Dict		       =							'''0'''
																				if aux == "1" and x == "1":
																									cont_bo += 1
															cont_loop += 1
										parity.append(cont_bo % 2	)

										qtd_bp += 1

					# Mount the message
					__magic_name__ :							int		       =							0  # parity bit counter
					for x in range(0,     size_par + len(UpperCAmelCase	)	):
										if data_ord[x] is None:
															data_out.append(str(parity[cont_bp]	)	)
															cont_bp += 1
										else:
															data_out.append(data_ord[x]	)

					return data_out







def 			lowerCAmelCase  (     UpperCAmelCase,     UpperCAmelCase	)  ->List[Any]:






					"""simple docstring"""






					__magic_name__ :							str		       =							[]
					# Parity bit counter
					__magic_name__ :							str		       =							0
					# Counter p data bit reading
					__magic_name__ :							Tuple		       =							0
					# list of parity received
					__magic_name__ :							List[str]		       =							[]
					__magic_name__ :							int		       =							[]

					for x in range(1,     len(UpperCAmelCase	) + 1	):
										# Performs a template of bit positions - who should be given,
										#  and who should be parity
										if qtd_bp < size_par and (np.log(UpperCAmelCase	) / np.log(2	)).is_integer():
															data_out_gab.append('''P'''	)
															__magic_name__ :							Optional[Any]		       =							qtd_bp + 1
										else:
															data_out_gab.append('''D'''	)

										# Sorts the data to the new output size
										if data_out_gab[-1] == "D":
															data_output.append(data[cont_data]	)
										else:
															parity_received.append(data[cont_data]	)
										cont_data += 1

					# -----------calculates the parity with the data
					__magic_name__ :							List[Any]		       =							[]
					__magic_name__ :							List[Any]		       =							[]
					__magic_name__ :							List[Any]		       =							[bin(UpperCAmelCase	)[2:] for x in range(1,     size_par + len(UpperCAmelCase	) + 1	)]

					#  sorted information data for the size of the output data
					__magic_name__ :							List[Any]		       =							[]
					# Data position feedback + parity
					__magic_name__ :							int		       =							[]
					# Parity bit counter
					__magic_name__ :							List[str]		       =							0
					# Counter p data bit reading
					__magic_name__ :							int		       =							0

					for x in range(1,     size_par + len(UpperCAmelCase	) + 1	):
										# Performs a template position of bits - who should be given,
										# and who should be parity
										if qtd_bp < size_par and (np.log(UpperCAmelCase	) / np.log(2	)).is_integer():
															data_out_gab.append('''P'''	)
															__magic_name__ :							Any		       =							qtd_bp + 1
										else:
															data_out_gab.append('''D'''	)

										# Sorts the data to the new output size
										if data_out_gab[-1] == "D":
															data_ord.append(data_output[cont_data]	)
															cont_data += 1
										else:
															data_ord.append(UpperCAmelCase	)

    # Calculates parity
					__magic_name__ :							List[Any]		       =							0  # parity bit counter
					for bp in range(1,     size_par + 1	):
										# Bit counter one for a certain parity
										__magic_name__ :							Dict		       =							0
										# Counter to control loop reading
										__magic_name__ :							Union[str, Any]		       =							0
										for x in data_ord:
															if x is not None:
																				try:
																									__magic_name__ :							int		       =							(bin_pos[cont_loop])[-1 * (bp)]
																				except IndexError:
																									__magic_name__ :							Dict		       =							'''0'''
																				if aux == "1" and x == "1":
																									cont_bo += 1
															cont_loop += 1
										parity.append(str(cont_bo % 2	)	)

										qtd_bp += 1

					# Mount the message
					__magic_name__ :							Tuple		       =							0  # Parity bit counter
					for x in range(0,     size_par + len(UpperCAmelCase	)	):
										if data_ord[x] is None:
															data_out.append(str(parity[cont_bp]	)	)
															cont_bp += 1
										else:
															data_out.append(data_ord[x]	)

					__magic_name__ :							List[str]		       =							parity_received == parity
					return data_output, ack









import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


lowercase_		=							get_logger(__name__)

# Datasets
lowercase_		=							'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'''
lowercase_		=							'''https://cdn-datasets.huggingface.co/datasets/datasets'''
lowercase_		=							'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'''

# Metrics
lowercase_		=							'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'''
lowercase_		=							'''https://cdn-datasets.huggingface.co/datasets/metric'''
lowercase_		=							'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'''

# Hub
lowercase_		=							os.environ.get('''HF_ENDPOINT''', '''https://huggingface.co''')
lowercase_		=							HF_ENDPOINT + '''/datasets/{repo_id}/resolve/{revision}/{path}'''
lowercase_		=							'''main'''

lowercase_		=							version.parse(platform.python_version())

if PY_VERSION < version.parse('''3.8'''):
				import importlib_metadata
else:
				import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
lowercase_		=							{'''1''', '''ON''', '''YES''', '''TRUE'''}
lowercase_		=							ENV_VARS_TRUE_VALUES.union({'''AUTO'''})


# Imports
lowercase_		=							version.parse(importlib_metadata.version('''dill'''))
lowercase_		=							version.parse(importlib_metadata.version('''pandas'''))
lowercase_		=							version.parse(importlib_metadata.version('''pyarrow'''))

lowercase_		=							os.environ.get('''USE_TF''', '''AUTO''').upper()
lowercase_		=							os.environ.get('''USE_TORCH''', '''AUTO''').upper()
lowercase_		=							os.environ.get('''USE_JAX''', '''AUTO''').upper()

lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
				lowercase_		=							importlib.util.find_spec('''torch''') is not None
				if TORCH_AVAILABLE:
								try:
												lowercase_		=							version.parse(importlib_metadata.version('''torch'''))
												logger.info(f"PyTorch version {TORCH_VERSION} available.")
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info('''Disabling PyTorch because USE_TF is set''')

lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
				lowercase_		=							importlib.util.find_spec('''tensorflow''') is not None
				if TF_AVAILABLE:
								# For the metadata, we have to look for both tensorflow and tensorflow-cpu
								for package in [
								    "tensorflow",
								    "tensorflow-cpu",
								    "tensorflow-gpu",
								    "tf-nightly",
								    "tf-nightly-cpu",
								    "tf-nightly-gpu",
								    "intel-tensorflow",
								    "tensorflow-rocm",
								    "tensorflow-macos",
								]:
												try:
																lowercase_		=							version.parse(importlib_metadata.version(package))
												except importlib_metadata.PackageNotFoundError:
																continue
												else:
																break
								else:
												lowercase_		=							False
				if TF_AVAILABLE:
								if TF_VERSION.major < 2:
												logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
												lowercase_		=							False
								else:
												logger.info(f"TensorFlow version {TF_VERSION} available.")
else:
				logger.info('''Disabling Tensorflow because USE_TORCH is set''')


lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
				lowercase_		=							importlib.util.find_spec('''jax''') is not None and importlib.util.find_spec('''jaxlib''') is not None
				if JAX_AVAILABLE:
								try:
												lowercase_		=							version.parse(importlib_metadata.version('''jax'''))
												logger.info(f"JAX version {JAX_VERSION} available.")
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info('''Disabling JAX because USE_JAX is set to False''')


lowercase_		=							os.environ.get('''USE_BEAM''', '''AUTO''').upper()
lowercase_		=							'''N/A'''
lowercase_		=							False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
				try:
								lowercase_		=							version.parse(importlib_metadata.version('''apache_beam'''))
								lowercase_		=							True
								logger.info(f"Apache Beam version {BEAM_VERSION} available.")
				except importlib_metadata.PackageNotFoundError:
								pass
else:
				logger.info('''Disabling Apache Beam because USE_BEAM is set to False''')


# Optional tools for data loading
lowercase_		=							importlib.util.find_spec('''sqlalchemy''') is not None

# Optional tools for feature decoding
lowercase_		=							importlib.util.find_spec('''PIL''') is not None
lowercase_		=							importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.0.31''')
lowercase_		=							importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.1.0''')

# Optional compression tools
lowercase_		=							importlib.util.find_spec('''rarfile''') is not None
lowercase_		=							importlib.util.find_spec('''zstandard''') is not None
lowercase_		=							importlib.util.find_spec('''lz4''') is not None
lowercase_		=							importlib.util.find_spec('''py7zr''') is not None


# Cache location
lowercase_		=							'''~/.cache'''
lowercase_		=							os.getenv('''XDG_CACHE_HOME''', DEFAULT_XDG_CACHE_HOME)
lowercase_		=							os.path.join(XDG_CACHE_HOME, '''huggingface''')
lowercase_		=							os.path.expanduser(os.getenv('''HF_HOME''', DEFAULT_HF_CACHE_HOME))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''datasets''')
lowercase_		=							Path(os.getenv('''HF_DATASETS_CACHE''', DEFAULT_HF_DATASETS_CACHE))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''metrics''')
lowercase_		=							Path(os.getenv('''HF_METRICS_CACHE''', DEFAULT_HF_METRICS_CACHE))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''modules''')
lowercase_		=							Path(os.getenv('''HF_MODULES_CACHE''', DEFAULT_HF_MODULES_CACHE))

lowercase_		=							'''downloads'''
lowercase_		=							os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
lowercase_		=							Path(os.getenv('''HF_DATASETS_DOWNLOADED_DATASETS_PATH''', DEFAULT_DOWNLOADED_DATASETS_PATH))

lowercase_		=							'''extracted'''
lowercase_		=							os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
lowercase_		=							Path(os.getenv('''HF_DATASETS_EXTRACTED_DATASETS_PATH''', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
lowercase_		=							(
    os.environ.get('''HF_UPDATE_DOWNLOAD_COUNTS''', '''AUTO''').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
lowercase_		=							1000

# Size of the preloaded record batch in `Dataset.__iter__`
lowercase_		=							10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
lowercase_		=							4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
lowercase_		=							'''500MB'''

# Parquet configuration
lowercase_		=							100
lowercase_		=							100
lowercase_		=							100

# Offline mode
lowercase_		=							os.environ.get('''HF_DATASETS_OFFLINE''', '''AUTO''').upper() in ENV_VARS_TRUE_VALUES

# In-memory
lowercase_		=							0  # Disabled
lowercase_		=							float(os.environ.get('''HF_DATASETS_IN_MEMORY_MAX_SIZE''', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
lowercase_		=							'''dataset.arrow'''
lowercase_		=							'''indices.arrow'''
lowercase_		=							'''state.json'''
lowercase_		=							'''dataset_info.json'''
lowercase_		=							'''dataset_infos.json'''
lowercase_		=							'''LICENSE'''
lowercase_		=							'''metric_info.json'''
lowercase_		=							'''dataset_dict.json'''

lowercase_		=							'''datasets_modules'''

lowercase_		=							255

# Streaming
lowercase_		=							20
lowercase_		=							5

# Datasets without script
lowercase_		=							200
lowercase_		=							10
lowercase_		=							200

# Progress bars
lowercase_		=							0.0_5  # 20 progress updates per sec



