





import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
				import os

				# The slow tests are often failing with OOM error on GPU
				# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
				# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
				lowercase_		=							'''platform'''

				import jax
				import jax.numpy as jnp

				from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer







def 			lowerCAmelCase  (     UpperCAmelCase,     UpperCAmelCase,     UpperCAmelCase=None,     UpperCAmelCase=None	)  ->Any:






					"""simple docstring"""






					if attention_mask is None:
										__magic_name__ :							int		       =							np.where(input_ids != config.pad_token_id,     1,     0	)
					return {
					    "input_ids": input_ids,
					    "attention_mask": attention_mask,
					}





@require_flax
class 		A__    :






	def __init__(     self    ,					lowerCamelCase    ,					lowerCamelCase=13    ,					lowerCamelCase=7    ,					lowerCamelCase=True    ,					lowerCamelCase=False    ,					lowerCamelCase=99    ,					lowerCamelCase=16    ,					lowerCamelCase=2    ,					lowerCamelCase=4    ,					lowerCamelCase=4    ,					lowerCamelCase="gelu"    ,					lowerCamelCase=0.1    ,					lowerCamelCase=0.1    ,					lowerCamelCase=20    ,					lowerCamelCase=2    ,					lowerCamelCase=1    ,					lowerCamelCase=0    ,					lowerCamelCase=16    ,					lowerCamelCase=16    ,					lowerCamelCase=0.0_2    ,					)				->       Any:




						"""simple docstring"""






						__magic_name__ :							List[Any]		       =							parent
						__magic_name__ :							Optional[int]		       =							batch_size
						__magic_name__ :							Tuple		       =							seq_length
						__magic_name__ :							Any		       =							is_training
						__magic_name__ :							List[str]		       =							use_labels
						__magic_name__ :							List[Any]		       =							vocab_size
						__magic_name__ :							int		       =							hidden_size
						__magic_name__ :							Any		       =							num_hidden_layers
						__magic_name__ :							Dict		       =							num_attention_heads
						__magic_name__ :							Optional[int]		       =							intermediate_size
						__magic_name__ :							int		       =							hidden_act
						__magic_name__ :							List[str]		       =							hidden_dropout_prob
						__magic_name__ :							Union[str, Any]		       =							attention_probs_dropout_prob
						__magic_name__ :							Optional[int]		       =							max_position_embeddings
						__magic_name__ :							Dict		       =							eos_token_id
						__magic_name__ :							List[Any]		       =							pad_token_id
						__magic_name__ :							Union[str, Any]		       =							bos_token_id
						__magic_name__ :							Optional[int]		       =							embed_dim
						__magic_name__ :							Union[str, Any]		       =							word_embed_proj_dim
						__magic_name__ :							List[Any]		       =							initializer_range
						__magic_name__ :							Union[str, Any]		       =							False






	def 			lowercase				(     self						)				->       List[Any]:




						"""simple docstring"""






						__magic_name__ :							str		       =							np.clip(ids_tensor([self.batch_size, self.seq_length - 1]    ,					self.vocab_size						)    ,					3    ,					self.vocab_size						)
						__magic_name__ :							Optional[int]		       =							np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)    ,					dtype=np.intaa						))    ,					-1						)

						__magic_name__ :							Any		       =							OPTConfig(
						    vocab_size=self.vocab_size    ,					hidden_size=self.hidden_size    ,					num_hidden_layers=self.num_hidden_layers    ,					num_attention_heads=self.num_attention_heads    ,					ffn_dim=self.intermediate_size    ,					dropout=self.hidden_dropout_prob    ,					attention_dropout=self.attention_probs_dropout_prob    ,					max_position_embeddings=self.max_position_embeddings    ,					eos_token_id=self.eos_token_id    ,					bos_token_id=self.bos_token_id    ,					pad_token_id=self.pad_token_id    ,					embed_dim=self.embed_dim    ,					is_encoder_decoder=_UpperCAmelCase    ,					word_embed_proj_dim=self.word_embed_proj_dim    ,					initializer_range=self.initializer_range    ,					use_cache=_UpperCAmelCase    ,					)
						__magic_name__ :							int		       =							prepare_opt_inputs_dict(_UpperCAmelCase    ,					_UpperCAmelCase						)
						return config, inputs_dict






	def 			lowercase				(     self						)				->       Optional[Any]:




						"""simple docstring"""






						__magic_name__       ,			__magic_name__ :							Optional[Any]		       =							self.prepare_config_and_inputs()
						return config, inputs_dict






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase						)				->       List[Any]:




						"""simple docstring"""






						__magic_name__ :							Union[str, Any]		       =							20
						__magic_name__ :							Dict		       =							model_class_name(_UpperCAmelCase						)

						__magic_name__ :							Optional[Any]		       =							inputs_dict['''input_ids''']
						__magic_name__ :							str		       =							inputs_dict['''attention_mask''']

						__magic_name__ :							Optional[int]		       =							model.init_cache(input_ids.shape[0]    ,					_UpperCAmelCase						)
						__magic_name__ :							List[str]		       =							jnp.ones((input_ids.shape[0], max_length)    ,					dtype='''i4'''						)

						__magic_name__ :							int		       =							jnp.broadcast_to(
						    jnp.arange(input_ids.shape[-1] - 1						)[None, :]    ,					(input_ids.shape[0], input_ids.shape[-1] - 1)    ,					)
						__magic_name__ :							Dict		       =							model(
						    input_ids[:, :-1]    ,					attention_mask=_UpperCAmelCase    ,					past_key_values=_UpperCAmelCase    ,					position_ids=_UpperCAmelCase    ,					)

						__magic_name__ :							Any		       =							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]    ,					dtype='''i4'''						)
						__magic_name__ :							Tuple		       =							model(
						    input_ids[:, -1:]    ,					attention_mask=_UpperCAmelCase    ,					past_key_values=outputs_cache.past_key_values    ,					position_ids=_UpperCAmelCase    ,					)

						__magic_name__ :							Union[str, Any]		       =							model(_UpperCAmelCase						)

						__magic_name__ :							str		       =							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
						self.parent.assertTrue(diff < 1e-3    ,					msg=F'''Max diff is {diff}'''						)






	def 			lowercase				(     self    ,					lowerCamelCase    ,					lowerCamelCase    ,					lowerCamelCase						)				->       Tuple:




						"""simple docstring"""






						__magic_name__ :							int		       =							20
						__magic_name__ :							int		       =							model_class_name(_UpperCAmelCase						)

						__magic_name__       ,			__magic_name__ :							Union[str, Any]		       =							(
						    inputs_dict['''input_ids'''],
						    inputs_dict['''attention_mask'''],
						)

						__magic_name__ :							str		       =							jnp.concatenate(
						    [
						        attention_mask,
						        jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])						),
						    ]    ,					axis=-1    ,					)

						__magic_name__ :							Dict		       =							model.init_cache(input_ids.shape[0]    ,					_UpperCAmelCase						)
						__magic_name__ :							Optional[Any]		       =							jnp.broadcast_to(
						    jnp.arange(input_ids.shape[-1] - 1						)[None, :]    ,					(input_ids.shape[0], input_ids.shape[-1] - 1)    ,					)

						__magic_name__ :							Any		       =							model(
						    input_ids[:, :-1]    ,					attention_mask=_UpperCAmelCase    ,					past_key_values=_UpperCAmelCase    ,					position_ids=_UpperCAmelCase    ,					)
						__magic_name__ :							List[str]		       =							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]    ,					dtype='''i4'''						)
						__magic_name__ :							int		       =							model(
						    input_ids[:, -1:]    ,					past_key_values=outputs_cache.past_key_values    ,					attention_mask=_UpperCAmelCase    ,					position_ids=_UpperCAmelCase    ,					)

						__magic_name__ :							Union[str, Any]		       =							model(_UpperCAmelCase    ,					attention_mask=_UpperCAmelCase						)

						__magic_name__ :							Optional[int]		       =							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
						self.parent.assertTrue(diff < 1e-3    ,					msg=F'''Max diff is {diff}'''						)





@require_flax
class 		A__    (       lowerCAmelCase__		,					unittest.TestCase		,					lowerCAmelCase__			):
	lowerCamelCase__		:			str       =(FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
	lowerCamelCase__		:			Optional[Any]       =() if is_flax_available() else ()






	def 			lowercase				(     self						)				->       Optional[int]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							FlaxOPTModelTester(self						)






	def 			lowercase				(     self						)				->       Union[str, Any]:




						"""simple docstring"""






						__magic_name__       ,			__magic_name__ :							Optional[Any]		       =							self.model_tester.prepare_config_and_inputs()
						for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward(_UpperCAmelCase    ,					_UpperCAmelCase    ,					_UpperCAmelCase						)






	def 			lowercase				(     self						)				->       Union[str, Any]:




						"""simple docstring"""






						__magic_name__       ,			__magic_name__ :							Optional[int]		       =							self.model_tester.prepare_config_and_inputs()
						for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward_with_attn_mask(_UpperCAmelCase    ,					_UpperCAmelCase    ,					_UpperCAmelCase						)






	@slow
	def 			lowercase				(     self						)				->       Tuple:




						"""simple docstring"""






						for model_class_name in self.all_model_classes:
											__magic_name__ :							Dict		       =							model_class_name.from_pretrained('''facebook/opt-125m'''						)
											__magic_name__ :							Optional[Any]		       =							np.ones((1, 1)						) * model.config.eos_token_id
											__magic_name__ :							List[str]		       =							model(_UpperCAmelCase						)
											self.assertIsNotNone(_UpperCAmelCase						)





@require_sentencepiece
@require_flax
class 		A__    (       unittest.TestCase			):






	@slow
	def 			lowercase				(     self						)				->       List[Any]:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							FlaxOPTModel.from_pretrained('''facebook/opt-350m'''						)
						__magic_name__ :							Dict		       =							jnp.array([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]						)
						__magic_name__ :							Optional[int]		       =							model(input_ids=_UpperCAmelCase						).last_hidden_state
						__magic_name__ :							Optional[Any]		       =							(1, 11, 512)
						self.assertEqual(output.shape    ,					_UpperCAmelCase						)
						__magic_name__ :							Optional[int]		       =							jnp.array(
						    [[-0.2_8_6_7, -1.9_2_5_6, -0.3_0_6_2], [-1.2_7_1_1, -0.1_3_3_7, -0.1_8_9_7], [0.4_1_0_9, 0.1_1_8_7, -1.3_1_4_2]]						)
						self.assertTrue(jnp.allclose(output[:, :3, :3]    ,					_UpperCAmelCase    ,					atol=4e-2						)						)





@require_flax
@slow
class 		A__    (       unittest.TestCase			):






	def 			lowercase				(     self						)				->       List[str]:




						"""simple docstring"""






						super().setUp()
						__magic_name__ :							Optional[Any]		       =							'''facebook/opt-350m'''






	def 			lowercase				(     self						)				->       str:




						"""simple docstring"""






						__magic_name__ :							Optional[int]		       =							FlaxOPTForCausalLM.from_pretrained(self.path_model						)
						__magic_name__ :							List[str]		       =							GPTaTokenizer.from_pretrained(self.path_model						)

						__magic_name__ :							Union[str, Any]		       =							[
						    '''Today is a beautiful day and I want to''',
						    '''In the city of''',
						    '''Paris is the capital of France and''',
						    '''Computers and mobile phones have taken''',
						]
						# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
						__magic_name__ :							str		       =							tokenizer(_UpperCAmelCase    ,					return_tensors='''jax'''    ,					padding=_UpperCAmelCase    ,					add_special_tokens=_UpperCAmelCase						)
						__magic_name__ :							Tuple		       =							model(inputs.input_ids    ,					attention_mask=inputs.attention_mask						)[0].mean(axis=-1						)
						__magic_name__ :							List[Any]		       =							jnp.array(
						    [
						        [1.3_8_5_1, -1_3.8_9_2_3, -1_0.5_2_2_9, -1_0.7_5_3_3, -0.2_3_0_9, -1_0.2_3_8_4, -0.5_3_6_5, -9.0_9_4_7, -5.1_6_7_0],
						        [-4.7_0_7_3, -1_0.6_2_7_6, -3.9_4_1_5, -2_1.5_2_4_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2],
						        [0.6_2_4_7, -3.4_2_2_9, -8.9_1_7_9, -1.4_2_9_7, -1_4.1_6_5_0, 1.4_1_4_6, -9.0_2_1_8, -0.2_7_0_3, -0.2_7_0_3],
						        [6.4_7_8_3, -1.9_9_1_3, -1_0.7_9_2_6, -2.3_3_3_6, 1.5_0_9_2, -0.9_9_7_4, -6.8_2_1_3, 1.3_4_7_7, 1.3_4_7_7],
						    ]						)
						self.assertTrue(jnp.allclose(_UpperCAmelCase    ,					_UpperCAmelCase    ,					atol=4e-2						)						)

						__magic_name__ :							Tuple		       =							jax.jit(_UpperCAmelCase						)
						__magic_name__ :							Optional[Any]		       =							model(inputs.input_ids    ,					attention_mask=inputs.attention_mask						)[0].mean(axis=-1						)
						self.assertTrue(jnp.allclose(_UpperCAmelCase    ,					_UpperCAmelCase    ,					atol=4e-2						)						)





@require_flax
@slow
class 		A__    (       unittest.TestCase			):






	@property
	def 			lowercase				(     self						)				->       Optional[int]:




						"""simple docstring"""






						return [
						    "Today is a beautiful day and I want",
						    "In the city of",
						    "Paris is the capital of France and",
						    "Computers and mobile phones have taken",
						]






	def 			lowercase				(     self						)				->       int:




						"""simple docstring"""






						__magic_name__ :							Dict		       =							'''facebook/opt-125m'''

						__magic_name__ :							List[Any]		       =							[
						    '''Today is a beautiful day and I want to''',
						    '''In the city of New York, the city''',
						    '''Paris is the capital of France and the capital''',
						    '''Computers and mobile phones have taken over the''',
						]

						__magic_name__ :							int		       =							[]

						__magic_name__ :							int		       =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase						)
						__magic_name__ :							Union[str, Any]		       =							GPTaTokenizer.from_pretrained(_UpperCAmelCase						)

						for prompt in self.prompts:
											__magic_name__ :							Union[str, Any]		       =							tokenizer(_UpperCAmelCase    ,					return_tensors='''jax'''						).input_ids

											__magic_name__ :							str		       =							model.generate(_UpperCAmelCase    ,					max_length=10						)
											__magic_name__ :							List[Any]		       =							generated_ids[0]

											__magic_name__ :							Union[str, Any]		       =							tokenizer.batch_decode(_UpperCAmelCase    ,					skip_special_tokens=_UpperCAmelCase						)
											predicted_outputs += generated_string

						self.assertListEqual(_UpperCAmelCase    ,					_UpperCAmelCase						)






	def 			lowercase				(     self						)				->       Dict:




						"""simple docstring"""






						__magic_name__ :							Tuple		       =							'''facebook/opt-350m'''

						__magic_name__ :							List[Any]		       =							[
						    '''Today is a beautiful day and I want to''',
						    '''In the city of San Francisco, the city''',
						    '''Paris is the capital of France and the capital''',
						    '''Computers and mobile phones have taken over the''',
						]

						__magic_name__ :							Tuple		       =							[]
						__magic_name__ :							Union[str, Any]		       =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase						)
						__magic_name__ :							int		       =							GPTaTokenizer.from_pretrained(_UpperCAmelCase						)

						for prompt in self.prompts:
											__magic_name__ :							List[str]		       =							tokenizer(_UpperCAmelCase    ,					return_tensors='''jax'''						).input_ids

											__magic_name__ :							Optional[int]		       =							model.generate(_UpperCAmelCase    ,					max_length=10						)
											__magic_name__ :							Optional[Any]		       =							generated_ids[0]

											__magic_name__ :							List[str]		       =							tokenizer.batch_decode(_UpperCAmelCase    ,					skip_special_tokens=_UpperCAmelCase						)
											predicted_outputs += generated_string

						self.assertListEqual(_UpperCAmelCase    ,					_UpperCAmelCase						)






	def 			lowercase				(     self						)				->       int:




						"""simple docstring"""






						__magic_name__ :							int		       =							'''facebook/opt-125m'''
						__magic_name__ :							Union[str, Any]		       =							[
						    '''Today is a beautiful day and I want to thank''',
						    '''In the city of Rome Canaver Canaver Canaver Canaver''',
						]
						__magic_name__ :							List[Any]		       =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase						)
						__magic_name__ :							Optional[Any]		       =							GPTaTokenizer.from_pretrained(_UpperCAmelCase						)
						__magic_name__ :							List[str]		       =							tokenizer(
						    [
						        '''Today is a beautiful day and I want to''',
						        '''In the city of''',
						    ]    ,					return_tensors='''jax'''    ,					padding=_UpperCAmelCase    ,					)

						__magic_name__ :							int		       =							jax.jit(model.generate						)

						__magic_name__ :							Optional[int]		       =							jit_generate(inputs['''input_ids''']    ,					attention_mask=inputs['''attention_mask''']						).sequences

						__magic_name__ :							Tuple		       =							tokenizer.batch_decode(_UpperCAmelCase    ,					skip_special_tokens=_UpperCAmelCase						)

						self.assertIsNotNone(_UpperCAmelCase    ,					_UpperCAmelCase						)






	def 			lowercase				(     self						)				->       Any:




						"""simple docstring"""






						__magic_name__ :							Tuple		       =							'''facebook/opt-350m'''

						__magic_name__ :							Optional[Any]		       =							GPTaTokenizer.from_pretrained(_UpperCAmelCase						)
						__magic_name__ :							Union[str, Any]		       =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase						)

						__magic_name__ :							Union[str, Any]		       =							'''left'''

						# use different length sentences to test batching
						__magic_name__ :							Optional[int]		       =							[
						    '''Hello, my dog is a little''',
						    '''Today, I''',
						]

						__magic_name__ :							Any		       =							tokenizer(_UpperCAmelCase    ,					return_tensors='''jax'''    ,					padding=_UpperCAmelCase						)
						__magic_name__ :							str		       =							inputs['''input_ids''']

						__magic_name__ :							Any		       =							model.generate(input_ids=_UpperCAmelCase    ,					attention_mask=inputs['''attention_mask''']    ,					trace=_UpperCAmelCase						)

						__magic_name__ :							Optional[int]		       =							tokenizer(sentences[0]    ,					return_tensors='''jax'''						).input_ids
						__magic_name__ :							Tuple		       =							model.generate(input_ids=_UpperCAmelCase						)

						__magic_name__ :							int		       =							inputs_non_padded.shape[-1] - inputs['''attention_mask'''][-1].sum()
						__magic_name__ :							str		       =							tokenizer(sentences[1]    ,					return_tensors='''jax'''						).input_ids
						__magic_name__ :							List[Any]		       =							model.generate(input_ids=_UpperCAmelCase    ,					max_length=model.config.max_length - num_paddings						)

						__magic_name__ :							int		       =							tokenizer.batch_decode(outputs[0]    ,					skip_special_tokens=_UpperCAmelCase						)
						__magic_name__ :							str		       =							tokenizer.decode(output_non_padded[0][0]    ,					skip_special_tokens=_UpperCAmelCase						)
						__magic_name__ :							Optional[Any]		       =							tokenizer.decode(output_padded[0][0]    ,					skip_special_tokens=_UpperCAmelCase						)

						__magic_name__ :							List[str]		       =							[
						    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
						    '''Today, I was in the middle of a conversation with a friend about the''',
						]
						self.assertListEqual(_UpperCAmelCase    ,					_UpperCAmelCase						)
						self.assertListEqual(_UpperCAmelCase    ,					[non_padded_sentence, padded_sentence]						)









import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


lowercase_		=							get_logger(__name__)

# Datasets
lowercase_		=							'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'''
lowercase_		=							'''https://cdn-datasets.huggingface.co/datasets/datasets'''
lowercase_		=							'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'''

# Metrics
lowercase_		=							'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'''
lowercase_		=							'''https://cdn-datasets.huggingface.co/datasets/metric'''
lowercase_		=							'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'''

# Hub
lowercase_		=							os.environ.get('''HF_ENDPOINT''', '''https://huggingface.co''')
lowercase_		=							HF_ENDPOINT + '''/datasets/{repo_id}/resolve/{revision}/{path}'''
lowercase_		=							'''main'''

lowercase_		=							version.parse(platform.python_version())

if PY_VERSION < version.parse('''3.8'''):
				import importlib_metadata
else:
				import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
lowercase_		=							{'''1''', '''ON''', '''YES''', '''TRUE'''}
lowercase_		=							ENV_VARS_TRUE_VALUES.union({'''AUTO'''})


# Imports
lowercase_		=							version.parse(importlib_metadata.version('''dill'''))
lowercase_		=							version.parse(importlib_metadata.version('''pandas'''))
lowercase_		=							version.parse(importlib_metadata.version('''pyarrow'''))

lowercase_		=							os.environ.get('''USE_TF''', '''AUTO''').upper()
lowercase_		=							os.environ.get('''USE_TORCH''', '''AUTO''').upper()
lowercase_		=							os.environ.get('''USE_JAX''', '''AUTO''').upper()

lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
				lowercase_		=							importlib.util.find_spec('''torch''') is not None
				if TORCH_AVAILABLE:
								try:
												lowercase_		=							version.parse(importlib_metadata.version('''torch'''))
												logger.info(f"PyTorch version {TORCH_VERSION} available.")
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info('''Disabling PyTorch because USE_TF is set''')

lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
				lowercase_		=							importlib.util.find_spec('''tensorflow''') is not None
				if TF_AVAILABLE:
								# For the metadata, we have to look for both tensorflow and tensorflow-cpu
								for package in [
								    "tensorflow",
								    "tensorflow-cpu",
								    "tensorflow-gpu",
								    "tf-nightly",
								    "tf-nightly-cpu",
								    "tf-nightly-gpu",
								    "intel-tensorflow",
								    "tensorflow-rocm",
								    "tensorflow-macos",
								]:
												try:
																lowercase_		=							version.parse(importlib_metadata.version(package))
												except importlib_metadata.PackageNotFoundError:
																continue
												else:
																break
								else:
												lowercase_		=							False
				if TF_AVAILABLE:
								if TF_VERSION.major < 2:
												logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
												lowercase_		=							False
								else:
												logger.info(f"TensorFlow version {TF_VERSION} available.")
else:
				logger.info('''Disabling Tensorflow because USE_TORCH is set''')


lowercase_		=							'''N/A'''
lowercase_		=							False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
				lowercase_		=							importlib.util.find_spec('''jax''') is not None and importlib.util.find_spec('''jaxlib''') is not None
				if JAX_AVAILABLE:
								try:
												lowercase_		=							version.parse(importlib_metadata.version('''jax'''))
												logger.info(f"JAX version {JAX_VERSION} available.")
								except importlib_metadata.PackageNotFoundError:
												pass
else:
				logger.info('''Disabling JAX because USE_JAX is set to False''')


lowercase_		=							os.environ.get('''USE_BEAM''', '''AUTO''').upper()
lowercase_		=							'''N/A'''
lowercase_		=							False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
				try:
								lowercase_		=							version.parse(importlib_metadata.version('''apache_beam'''))
								lowercase_		=							True
								logger.info(f"Apache Beam version {BEAM_VERSION} available.")
				except importlib_metadata.PackageNotFoundError:
								pass
else:
				logger.info('''Disabling Apache Beam because USE_BEAM is set to False''')


# Optional tools for data loading
lowercase_		=							importlib.util.find_spec('''sqlalchemy''') is not None

# Optional tools for feature decoding
lowercase_		=							importlib.util.find_spec('''PIL''') is not None
lowercase_		=							importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.0.31''')
lowercase_		=							importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.1.0''')

# Optional compression tools
lowercase_		=							importlib.util.find_spec('''rarfile''') is not None
lowercase_		=							importlib.util.find_spec('''zstandard''') is not None
lowercase_		=							importlib.util.find_spec('''lz4''') is not None
lowercase_		=							importlib.util.find_spec('''py7zr''') is not None


# Cache location
lowercase_		=							'''~/.cache'''
lowercase_		=							os.getenv('''XDG_CACHE_HOME''', DEFAULT_XDG_CACHE_HOME)
lowercase_		=							os.path.join(XDG_CACHE_HOME, '''huggingface''')
lowercase_		=							os.path.expanduser(os.getenv('''HF_HOME''', DEFAULT_HF_CACHE_HOME))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''datasets''')
lowercase_		=							Path(os.getenv('''HF_DATASETS_CACHE''', DEFAULT_HF_DATASETS_CACHE))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''metrics''')
lowercase_		=							Path(os.getenv('''HF_METRICS_CACHE''', DEFAULT_HF_METRICS_CACHE))

lowercase_		=							os.path.join(HF_CACHE_HOME, '''modules''')
lowercase_		=							Path(os.getenv('''HF_MODULES_CACHE''', DEFAULT_HF_MODULES_CACHE))

lowercase_		=							'''downloads'''
lowercase_		=							os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
lowercase_		=							Path(os.getenv('''HF_DATASETS_DOWNLOADED_DATASETS_PATH''', DEFAULT_DOWNLOADED_DATASETS_PATH))

lowercase_		=							'''extracted'''
lowercase_		=							os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
lowercase_		=							Path(os.getenv('''HF_DATASETS_EXTRACTED_DATASETS_PATH''', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
lowercase_		=							(
    os.environ.get('''HF_UPDATE_DOWNLOAD_COUNTS''', '''AUTO''').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
lowercase_		=							1000

# Size of the preloaded record batch in `Dataset.__iter__`
lowercase_		=							10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
lowercase_		=							4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
lowercase_		=							'''500MB'''

# Parquet configuration
lowercase_		=							100
lowercase_		=							100
lowercase_		=							100

# Offline mode
lowercase_		=							os.environ.get('''HF_DATASETS_OFFLINE''', '''AUTO''').upper() in ENV_VARS_TRUE_VALUES

# In-memory
lowercase_		=							0  # Disabled
lowercase_		=							float(os.environ.get('''HF_DATASETS_IN_MEMORY_MAX_SIZE''', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
lowercase_		=							'''dataset.arrow'''
lowercase_		=							'''indices.arrow'''
lowercase_		=							'''state.json'''
lowercase_		=							'''dataset_info.json'''
lowercase_		=							'''dataset_infos.json'''
lowercase_		=							'''LICENSE'''
lowercase_		=							'''metric_info.json'''
lowercase_		=							'''dataset_dict.json'''

lowercase_		=							'''datasets_modules'''

lowercase_		=							255

# Streaming
lowercase_		=							20
lowercase_		=							5

# Datasets without script
lowercase_		=							200
lowercase_		=							10
lowercase_		=							200

# Progress bars
lowercase_		=							0.0_5  # 20 progress updates per sec



